diff options
450 files changed, 7388 insertions, 3621 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 45700cb4ac..3090deef78 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -194,13 +194,15 @@ task: name: 'ARM [unit tests, no functional tests] [bullseye]' << : *GLOBAL_TASK_TEMPLATE arm_container: - image: debian:bullseye cpu: 2 memory: 8G - # docker_arguments: # Can use dockerfile after https://github.com/cirruslabs/cirrus-ci-docs/issues/1154 + dockerfile: ci/test_imagefile + docker_arguments: + CI_IMAGE_NAME_TAG: debian:bullseye + FILE_ENV: "./ci/test/00_setup_env_arm.sh" + << : *CREDITS_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_arm.sh" QEMU_USER_CMD: "" # Disable qemu and run the test natively task: @@ -215,11 +217,11 @@ task: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: '32-bit + dash [gui] [CentOS 8]' + name: '32-bit + dash [gui] [CentOS 9]' << : *GLOBAL_TASK_TEMPLATE container: docker_arguments: - CI_IMAGE_NAME_TAG: quay.io/centos/centos:stream8 + CI_IMAGE_NAME_TAG: quay.io/centos/centos:stream9 FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" # For faster CI feedback, immediately schedule one task that runs all tests << : *CREDITS_TEMPLATE @@ -241,27 +243,28 @@ task: name: '[TSan, depends, gui] [lunar]' << : *GLOBAL_TASK_TEMPLATE container: - cpu: 6 # Increase CPU and Memory to avoid timeout - memory: 24G + cpu: 4 + memory: 16G # The default memory is too small, so double everything docker_arguments: CI_IMAGE_NAME_TAG: ubuntu:lunar FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" + << : *CREDITS_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: '[MSan, depends] [lunar]' + name: '[MSan, depends] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:lunar + CI_IMAGE_NAME_TAG: ubuntu:jammy FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan task: - name: '[ASan + LSan + UBSan + integer, no depends, USDT] [jammy]' + name: '[ASan + LSan + UBSan + integer, no depends, USDT] [lunar]' << : *GLOBAL_TASK_TEMPLATE # We can't use a 'container' for the USDT interface tests as the CirrusCI # containers don't have privileges to hook into bitcoind. CirrusCI uses @@ -296,7 +299,7 @@ task: << : *GLOBAL_TASK_TEMPLATE container: cpu: 4 - memory: 16G # The default memory is sometimes just a bit too small, so double everything + memory: 16G # The default memory is too small, so double everything docker_arguments: CI_IMAGE_NAME_TAG: ubuntu:focal FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" @@ -346,21 +349,3 @@ task: CI_USE_APT_INSTALL: "no" PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh" - -task: - name: 'ARM64 Android APK [jammy]' - << : *CONTAINER_DEPENDS_TEMPLATE - container: - docker_arguments: - CI_IMAGE_NAME_TAG: ubuntu:jammy - FILE_ENV: "./ci/test/00_setup_env_android.sh" - << : *CREDITS_TEMPLATE - android_sdk_cache: - folder: "depends/SDKs/android" - fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.2.8568313" - depends_sources_cache: - folder: "depends/sources" - fingerprint_script: git rev-parse HEAD:depends/packages - << : *MAIN_TEMPLATE - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV diff --git a/.style.yapf b/.style.yapf index 69d8c6aee4..350ac63855 100644 --- a/.style.yapf +++ b/.style.yapf @@ -107,7 +107,7 @@ each_dict_entry_on_separate_line=True i18n_comment= # The i18n function call names. The presence of this function stops -# reformattting on that line, because the string it has cannot be moved +# reformatting on that line, because the string it has cannot be moved # away from the i18n comment. i18n_function_call= diff --git a/autogen.sh b/autogen.sh index de16260b56..d0ac7ef7ed 100755 --- a/autogen.sh +++ b/autogen.sh @@ -14,3 +14,12 @@ fi command -v autoreconf >/dev/null || \ (echo "configuration failed, please install autoconf first" && exit 1) autoreconf --install --force --warnings=all + +if expr "'$(build-aux/config.guess --timestamp)" \< "'$(depends/config.guess --timestamp)" > /dev/null; then + cp depends/config.guess build-aux + cp depends/config.guess src/secp256k1/build-aux +fi +if expr "'$(build-aux/config.sub --timestamp)" \< "'$(depends/config.sub --timestamp)" > /dev/null; then + cp depends/config.sub build-aux + cp depends/config.sub src/secp256k1/build-aux +fi diff --git a/build-aux/m4/l_filesystem.m4 b/build-aux/m4/l_filesystem.m4 deleted file mode 100644 index ca3a0cd41c..0000000000 --- a/build-aux/m4/l_filesystem.m4 +++ /dev/null @@ -1,47 +0,0 @@ -dnl Copyright (c) 2022 The Bitcoin Core developers -dnl Distributed under the MIT software license, see the accompanying -dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# GCC 8.1 and earlier requires -lstdc++fs -# Clang 8.0.0 (libc++) and earlier requires -lc++fs - -m4_define([_CHECK_FILESYSTEM_testbody], [[ - #include <filesystem> - - namespace fs = std::filesystem; - - int main() { - (void)fs::current_path().root_name(); - return 0; - } -]]) - -AC_DEFUN([CHECK_FILESYSTEM], [ - - AC_LANG_PUSH(C++) - - AC_MSG_CHECKING([whether std::filesystem can be used without link library]) - - AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_RESULT([no]) - SAVED_LIBS="$LIBS" - LIBS="$SAVED_LIBS -lstdc++fs" - AC_MSG_CHECKING([whether std::filesystem needs -lstdc++fs]) - AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_RESULT([no]) - AC_MSG_CHECKING([whether std::filesystem needs -lc++fs]) - LIBS="$SAVED_LIBS -lc++fs" - AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_FAILURE([cannot figure out how to use std::filesystem]) - ]) - ]) - ]) - - AC_LANG_POP -]) diff --git a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in index fc9d7cbed6..a5702a83ba 100644 --- a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in +++ b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in @@ -12,7 +12,7 @@ @SOURCE_FILES@ </ItemGroup> <ItemGroup> - <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> + <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj"> <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj"> diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index 0d6358e0d0..20cdb7bb6e 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -13,7 +13,7 @@ <ResourceCompile Include="..\..\src\qt\res\bitcoin-qt-res.rc" /> </ItemGroup> <ItemGroup> - <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> + <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj"> <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_cli\libbitcoin_cli.vcxproj"> diff --git a/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj b/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj index 4e9b4916a0..52585b98f9 100644 --- a/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj +++ b/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj @@ -12,7 +12,7 @@ <ClCompile Include="..\..\src\bitcoin-tx.cpp" /> </ItemGroup> <ItemGroup> - <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> + <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj"> <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj"> diff --git a/build_msvc/bitcoin-util/bitcoin-util.vcxproj b/build_msvc/bitcoin-util/bitcoin-util.vcxproj index 8a0964824b..4ea27fe439 100644 --- a/build_msvc/bitcoin-util/bitcoin-util.vcxproj +++ b/build_msvc/bitcoin-util/bitcoin-util.vcxproj @@ -12,7 +12,7 @@ <ClCompile Include="..\..\src\bitcoin-util.cpp" /> </ItemGroup> <ItemGroup> - <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> + <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj"> <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj"> diff --git a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj index 2ac0be9814..56d88d6a44 100644 --- a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj +++ b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj @@ -15,7 +15,7 @@ </ClCompile> </ItemGroup> <ItemGroup> - <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> + <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj"> <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj"> diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index 2a1ccf58fe..0931bf5dfe 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -2,7 +2,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.28803.452 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoinconsensus", "libbitcoinconsensus\libbitcoinconsensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_consensus", "libbitcoin_consensus\libbitcoin_consensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoind", "bitcoind\bitcoind.vcxproj", "{D4513DDF-6013-44DC-ADCC-12EAF6D1F038}" EndProject diff --git a/build_msvc/bitcoin_config.h.in b/build_msvc/bitcoin_config.h.in index 02d8fc41c2..1716647486 100644 --- a/build_msvc/bitcoin_config.h.in +++ b/build_msvc/bitcoin_config.h.in @@ -38,15 +38,12 @@ /* Define to 1 to enable SQLite wallet */ #define USE_SQLITE 1 -/* Define to 1 to enable ZMQ functions */ +/* Define this symbol to enable ZMQ functions */ #define ENABLE_ZMQ 1 /* define if external signer support is enabled (requires Boost::Process) */ #define ENABLE_EXTERNAL_SIGNER /**/ -/* Define this symbol if the consensus lib has been built */ -#define HAVE_CONSENSUS_LIB 1 - /* Define to 1 if you have the declaration of `be16toh', and to 0 if you don't. */ #define HAVE_DECL_BE16TOH 0 diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index b1204d0d5d..bb61865e14 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -15,7 +15,7 @@ </ClCompile> </ItemGroup> <ItemGroup> - <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> + <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj"> <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj"> diff --git a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj b/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj index 4cb0bdc902..95fdcdb79b 100644 --- a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj +++ b/build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj @@ -11,15 +11,6 @@ <ClCompile Include="..\..\src\arith_uint256.cpp" /> <ClCompile Include="..\..\src\consensus\merkle.cpp" /> <ClCompile Include="..\..\src\consensus\tx_check.cpp" /> - <ClCompile Include="..\..\src\crypto\aes.cpp" /> - <ClCompile Include="..\..\src\crypto\chacha20.cpp" /> - <ClCompile Include="..\..\src\crypto\hmac_sha256.cpp" /> - <ClCompile Include="..\..\src\crypto\hmac_sha512.cpp" /> - <ClCompile Include="..\..\src\crypto\ripemd160.cpp" /> - <ClCompile Include="..\..\src\crypto\sha1.cpp" /> - <ClCompile Include="..\..\src\crypto\sha256.cpp" /> - <ClCompile Include="..\..\src\crypto\sha256_sse4.cpp" /> - <ClCompile Include="..\..\src\crypto\sha512.cpp" /> <ClCompile Include="..\..\src\hash.cpp" /> <ClCompile Include="..\..\src\primitives\block.cpp" /> <ClCompile Include="..\..\src\primitives\transaction.cpp" /> diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index 0b90f341a7..585c28e503 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -15,6 +15,7 @@ <ItemDefinitionGroup> <ClCompile> <PreprocessorDefinitions>ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <UndefinePreprocessorDefinitions>USE_ASM_X86_64;%(UndefinePreprocessorDefinitions)</UndefinePreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src\secp256k1;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <DisableSpecificWarnings>4146;4244;4267;4334</DisableSpecificWarnings> </ClCompile> diff --git a/build_msvc/libsecp256k1_config.h b/build_msvc/libsecp256k1_config.h deleted file mode 100644 index 2b1a980e27..0000000000 --- a/build_msvc/libsecp256k1_config.h +++ /dev/null @@ -1,15 +0,0 @@ -/********************************************************************** - * Copyright (c) 2013, 2014 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php.* - **********************************************************************/ - -#ifndef BITCOIN_LIBSECP256K1_CONFIG_H -#define BITCOIN_LIBSECP256K1_CONFIG_H - -#undef USE_ASM_X86_64 - -#define ECMULT_GEN_PREC_BITS 4 -#define ECMULT_WINDOW_SIZE 15 - -#endif // BITCOIN_LIBSECP256K1_CONFIG_H diff --git a/build_msvc/libtest_util/libtest_util.vcxproj.in b/build_msvc/libtest_util/libtest_util.vcxproj.in index b5e844010e..64cfa82dcc 100644 --- a/build_msvc/libtest_util/libtest_util.vcxproj.in +++ b/build_msvc/libtest_util/libtest_util.vcxproj.in @@ -8,6 +8,7 @@ <ConfigurationType>StaticLibrary</ConfigurationType> </PropertyGroup> <ItemGroup> + <ClCompile Include="..\..\src\wallet\test\util.cpp" /> @SOURCE_FILES@ </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index e02e3abdfa..9484f0cb89 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -111,7 +111,6 @@ def main(): set_properties(vcxproj_filename, '@SOURCE_FILES@\n', content) parse_config_into_btc_config() copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h')) - copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h')) if __name__ == '__main__': main() diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index 3a2540d549..3776317fc7 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -10,13 +10,18 @@ <ItemGroup> <ClCompile Include="..\..\src\init\bitcoin-qt.cpp" /> <ClCompile Include="..\..\src\test\util\setup_common.cpp" /> + <ClCompile Include="..\..\src\wallet\test\util.cpp"> + <ObjectFileName>$(IntDir)wallet_test_util.obj</ObjectFileName> + </ClCompile> <ClCompile Include="..\..\src\qt\test\addressbooktests.cpp" /> <ClCompile Include="..\..\src\qt\test\apptests.cpp" /> <ClCompile Include="..\..\src\qt\test\optiontests.cpp" /> <ClCompile Include="..\..\src\qt\test\rpcnestedtests.cpp" /> <ClCompile Include="..\..\src\qt\test\test_main.cpp" /> <ClCompile Include="..\..\src\qt\test\uritests.cpp" /> - <ClCompile Include="..\..\src\qt\test\util.cpp" /> + <ClCompile Include="..\..\src\qt\test\util.cpp"> + <ObjectFileName>$(IntDir)qt_test_util.obj</ObjectFileName> + </ClCompile> <ClCompile Include="..\..\src\qt\test\wallettests.cpp" /> <ClCompile Include="..\..\src\wallet\test\wallet_test_fixture.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_addressbooktests.cpp" /> @@ -27,7 +32,7 @@ <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_wallettests.cpp" /> </ItemGroup> <ItemGroup> - <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> + <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj"> <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_cli\libbitcoin_cli.vcxproj"> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 4182448ec3..c0c84fc6f1 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -22,7 +22,7 @@ <ProjectReference Include="..\libminisketch\libminisketch.vcxproj"> <Project>{542007e3-be0d-4b0d-a6b0-aa8813e2558d}</Project> </ProjectReference> - <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> + <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj"> <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_cli\libbitcoin_cli.vcxproj"> diff --git a/ci/README.md b/ci/README.md index de798607df..b4158d0183 100644 --- a/ci/README.md +++ b/ci/README.md @@ -14,16 +14,10 @@ testing compared to other parts of the codebase. If you want to keep the work tr system in a virtual machine with a Linux operating system of your choice. To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage -requires `docker` to be installed. To install all requirements on Ubuntu, run +requires `bash`, `docker`, and `python3` to be installed. To install all requirements on Ubuntu, run ``` -sudo apt install docker.io bash -``` - -To run the default test stage, - -``` -./ci/test_run_all.sh +sudo apt install bash docker.io python3 ``` To run the test stage with a specific configuration, diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index f7147582dc..737cf3e2fb 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -35,6 +35,7 @@ fi ${CI_RETRY_EXE} pip3 install codespell==2.2.1 ${CI_RETRY_EXE} pip3 install flake8==5.0.4 +${CI_RETRY_EXE} pip3 install lief==0.13.1 ${CI_RETRY_EXE} pip3 install mypy==0.971 ${CI_RETRY_EXE} pip3 install pyzmq==24.0.1 ${CI_RETRY_EXE} pip3 install vulture==2.6 diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 4a54f47b03..69fd05051e 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -6,6 +6,8 @@ export LC_ALL=C.UTF-8 +set -ex + # The root dir. # The ci system copies this folder. BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) @@ -37,7 +39,6 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} export RUN_TIDY=${RUN_TIDY:-false} -export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # By how much to scale the test_runner timeouts (option --timeout-factor). # This is needed because some ci machines have slow CPU or disk, so sanitizers # might be slow or a reindex might be waiting on disk IO. @@ -45,8 +46,6 @@ export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-40} export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} -export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} -export CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG:-ubuntu:20.04} # Randomize test order. # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 544d6150a6..606c28e252 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -8,12 +8,10 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos -export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream8" -# Use minimum supported python3.8 and gcc-8, see doc/dependencies.md -export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python38 python38-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" +export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream9" +export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison util-linux" export PIP_PACKAGES="pyzmq" export GOAL="install" -export NO_WERROR=1 # GCC 8 +export NO_WERROR=1 # Suppress error: #warning _FORTIFY_SOURCE > 2 is treated like 2 on this platform [-Werror=cpp] export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports" export CONFIG_SHELL="/bin/dash" -export TEST_RUNNER_ENV="LC_ALL=en_US.UTF-8" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 9e3ea0d383..8ab4d54d31 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -12,6 +12,7 @@ export CI_IMAGE_NAME_TAG=ubuntu:20.04 export PACKAGES="cmake python3 llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" +export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' \ +LDFLAGS='--rtlib=compiler-rt -lgcc_s' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE'" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" export TEST_RUNNER_EXTRA="--nosandbox" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 8701d383dd..4de9511630 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -18,4 +18,6 @@ export PACKAGES="systemtap-sdt-dev clang-16 llvm-16 libclang-rt-16-dev python3-z export CI_IMAGE_NAME_TAG=ubuntu:23.04 # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang-16 CXX=clang++-16" +export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \ +CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ +--with-sanitizers=address,float-divide-by-zero,integer,undefined CC=clang-16 CXX=clang++-16" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 05cb45c2d8..298eb11da3 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -14,5 +14,6 @@ export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,float-divide-by-zero,integer \ +CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 35a0de8034..1f9adc0682 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -6,18 +6,18 @@ export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" +LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" -LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_fuzz_msan" -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" +export PACKAGES="cmake ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index bdb9bd7b5d..34d60c6c4c 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2022 The Bitcoin Core developers +# Copyright (c) 2020-2023 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). -LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" +LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" -LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" +export PACKAGES="cmake ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index b86fa6f321..8e3c935c6f 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -8,9 +8,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel export CI_IMAGE_NAME_TAG="ubuntu:20.04" -# Use minimum supported python3.8 and clang-8, see doc/dependencies.md -export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" -export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" +# Use minimum supported python3.8 and clang-10, see doc/dependencies.md +export PACKAGES="python3-zmq clang-10 llvm-10 libc++abi-10-dev libc++-10-dev" +export DEP_OPTS="NO_WALLET=1 CC=clang-10 CXX='clang++-10 -stdlib=libc++'" export GOAL="install" -export NO_WERROR=1 -export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" +export BITCOIN_CONFIG="--enable-reduce-exports --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 59d8c7c871..3a1d7808f1 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -8,14 +8,14 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 export CI_IMAGE_NAME_TAG="ubuntu:20.04" -# Use minimum supported python3.8 and gcc-8 (or best-effort gcc-9), see doc/dependencies.md +# Use minimum supported python3.8 and gcc-9, see doc/dependencies.md export PACKAGES="gcc-9 g++-9 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-9 CXX=g++-9" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export NO_WERROR=1 +export NO_WERROR=1 # -Werror=maybe-uninitialized export DOWNLOAD_PREVIOUS_RELEASES="true" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \ ---enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports --enable-debug \ +CFLAGS=\"-g0 -O2 -funsigned-char\" CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 2fa61b8465..920180a274 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2022 The Bitcoin Core developers +# Copyright (c) 2023 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="ubuntu:lunar" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). export CONTAINER_NAME=ci_native_tidy -export PACKAGES="clang-16 libclang-16-dev llvm-16-dev libomp-16-dev clang-tidy-16 bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export PACKAGES="clang-16 libclang-16-dev llvm-16-dev libomp-16-dev clang-tidy-16 jq bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index e3fa7ab777..8ebb1fa563 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2019-2023 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 641ff964f3..230288bc6c 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -6,6 +6,8 @@ export LC_ALL=C.UTF-8 +set -ex + CFG_DONE="ci.base-install-done" # Use a global git setting to remember whether this script ran to avoid running it twice if [ "$(git config --global ${CFG_DONE})" == "true" ]; then @@ -18,8 +20,8 @@ if [ -n "$DPKG_ADD_ARCH" ]; then fi if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - ${CI_RETRY_EXE} bash -c "dnf -y install epel-release" - ${CI_RETRY_EXE} bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES" + bash -c "dnf -y install epel-release" + bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES" elif [ "$CI_USE_APT_INSTALL" != "no" ]; then if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list @@ -40,11 +42,32 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - update-alternatives --install /usr/bin/clang++ clang++ "$(which clang++-16)" 100 - update-alternatives --install /usr/bin/clang clang "$(which clang-16)" 100 - git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.1 "${BASE_SCRATCH_DIR}"/msan/llvm-project - cmake -B "${BASE_SCRATCH_DIR}"/msan/build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DLIBCXX_ENABLE_DEBUG_MODE=ON -DLIBCXX_ENABLE_ASSERTIONS=ON -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes - make -C "${BASE_SCRATCH_DIR}"/msan/build/ "$MAKEJOBS" + git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.5 "${BASE_SCRATCH_DIR}"/msan/llvm-project + + cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/clang_build/ -DLLVM_ENABLE_PROJECTS="clang" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \ + -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/llvm + + ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ "$MAKEJOBS" + ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ install-runtimes + + update-alternatives --install /usr/bin/clang++ clang++ "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang++ 100 + update-alternatives --install /usr/bin/clang clang "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang 100 + + cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/cxx_build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=MemoryWithOrigins \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXX_ENABLE_DEBUG_MODE=ON \ + -DLIBCXX_ENABLE_ASSERTIONS=ON \ + -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes + + ninja -C "${BASE_SCRATCH_DIR}"/msan/cxx_build/ "$MAKEJOBS" fi if [[ "${RUN_TIDY}" == "true" ]]; then diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 1024222e96..626461df03 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -18,7 +18,6 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order= export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" -env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) CI_CONTAINER_CAP="--cap-add SYS_PTRACE" fi @@ -27,6 +26,9 @@ export P_CI_DIR="$PWD" export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/" if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then + # Export all env vars to avoid missing some. + # Though, exclude those with newlines to avoid parsing problems. + python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value and "HOME" not in key]' | tee /tmp/env echo "Creating $CI_IMAGE_NAME_TAG container to run in" DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \ --file "${BASE_ROOT_DIR}/ci/test_imagefile" \ @@ -40,7 +42,9 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then echo "Restart docker before run to stop and clear all containers started with --rm" - systemctl restart docker + podman container stop --all # Similar to "systemctl restart docker" + echo "Prune all dangling images" + docker image prune --force fi # shellcheck disable=SC2086 @@ -72,39 +76,3 @@ CI_EXEC rsync --archive --stats --human-readable /ro_base/ "${BASE_ROOT_DIR}" || CI_EXEC git config --global --add safe.directory \"*\" CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" - -if [ "$CI_OS_NAME" == "macos" ]; then - top -l 1 -s 0 | awk ' /PhysMem/ {print}' - echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" -else - CI_EXEC free -m -h - CI_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) - CI_EXEC echo "$(lscpu | grep Endian)" -fi -CI_EXEC echo "Free disk space:" -CI_EXEC df -h - -if [ "$RUN_FUZZ_TESTS" = "true" ]; then - export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ - if [ ! -d "$DIR_FUZZ_IN" ]; then - CI_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}" - fi -elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ - if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then - CI_EXEC mkdir -p "$DIR_UNIT_TEST_DATA" - CI_EXEC curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" - fi -fi - -CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" - -if [ "$USE_BUSY_BOX" = "true" ]; then - echo "Setup to use BusyBox utils" - # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) - # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) - # shellcheck disable=SC1010 - CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \)\; do ln -s \$\(command -v busybox\) "${BINS_SCRATCH_DIR}/\$util"\; done - # Print BusyBox version - CI_EXEC patch --help -fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh deleted file mode 100755 index 199cdd64a7..0000000000 --- a/ci/test/05_before_script.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -# Make sure default datadir does not exist and is never read by creating a dummy file -if [ "$CI_OS_NAME" == "macos" ]; then - echo > "${HOME}/Library/Application Support/Bitcoin" -else - CI_EXEC echo \> \$HOME/.bitcoin -fi - -if [ -z "$NO_DEPENDS" ]; then - if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - # CentOS has problems building the depends if the config shell is not explicitly set - # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to - # an error as the first command is executed) - SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/dash" - else - SHELL_OPTS="CONFIG_SHELL=" - fi - CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" LOG=1 -fi -if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then - CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" -fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh deleted file mode 100755 index 5856d33d2d..0000000000 --- a/ci/test/06_script_a.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" -if [ -z "$NO_DEPENDS" ]; then - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" -fi -if [ -z "$NO_WERROR" ]; then - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" -fi - -CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" -PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" - -if [ -n "$ANDROID_TOOLS_URL" ]; then - CI_EXEC make distclean || true - CI_EXEC ./autogen.sh - CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" - CI_EXEC "${PRINT_CCACHE_STATISTICS}" - exit 0 -fi - -BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" - -if [ -n "$CONFIG_SHELL" ]; then - CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" -else - CI_EXEC ./autogen.sh -fi - -CI_EXEC mkdir -p "${BASE_BUILD_DIR}" -export P_CI_DIR="${BASE_BUILD_DIR}" - -CI_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - -CI_EXEC make distdir VERSION="$HOST" - -export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST" - -CI_EXEC ./configure --cache-file=../config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - -set -o errtrace -trap 'CI_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR - -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - # MemorySanitizer (MSAN) does not support tracking memory initialization done by - # using the Linux getrandom syscall. Avoid using getrandom by undefining - # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for - # details. - CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' -fi - -if [[ "${RUN_TIDY}" == "true" ]]; then - MAYBE_BEAR="bear --config src/.bear-tidy-config" - MAYBE_TOKEN="--" -fi - -CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) - -CI_EXEC "${PRINT_CCACHE_STATISTICS}" -CI_EXEC du -sh "${DEPENDS_DIR}"/*/ -CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index e0fae18b8e..4c8b07c6a5 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -6,101 +6,169 @@ export LC_ALL=C.UTF-8 +set -ex + +if [ "$CI_OS_NAME" == "macos" ]; then + top -l 1 -s 0 | awk ' /PhysMem/ {print}' + echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" +else + free -m -h + echo "Number of CPUs (nproc): $(nproc)" + lscpu | grep Endian +fi +echo "Free disk space:" +df -h + +if [ "$RUN_FUZZ_TESTS" = "true" ]; then + export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + if [ ! -d "$DIR_FUZZ_IN" ]; then + git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}" + fi + ( + cd "${DIR_QA_ASSETS}" + echo "Using qa-assets repo from commit ..." + git log -1 + ) +elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then + export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ + if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then + mkdir -p "$DIR_UNIT_TEST_DATA" + curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" + fi +fi + +mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" + +if [ "$USE_BUSY_BOX" = "true" ]; then + echo "Setup to use BusyBox utils" + # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) + # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) + for util in $(busybox --list | grep -v "^ar$" | grep -v "^tar$" ); do ln -s "$(command -v busybox)" "${BINS_SCRATCH_DIR}/$util"; done + # Print BusyBox version + patch --help +fi + +# Make sure default datadir does not exist and is never read by creating a dummy file +if [ "$CI_OS_NAME" == "macos" ]; then + echo > "${HOME}/Library/Application Support/Bitcoin" +else + echo > "${HOME}/.bitcoin" +fi + +if [ -z "$NO_DEPENDS" ]; then + if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then + SHELL_OPTS="CONFIG_SHELL=/bin/dash" + else + SHELL_OPTS="CONFIG_SHELL=" + fi + bash -c "$SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS LOG=1" +fi +if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then + test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" +fi + +BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" +if [ -z "$NO_DEPENDS" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" +fi +if [ -z "$NO_WERROR" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" +fi + +ccache --zero-stats --max-size="${CCACHE_SIZE}" +PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" + +if [ -n "$ANDROID_TOOLS_URL" ]; then + make distclean || true + ./autogen.sh + bash -c "./configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false) + make "${MAKEJOBS}" && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk + bash -c "${PRINT_CCACHE_STATISTICS}" + exit 0 +fi + +BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" + +if [ -n "$CONFIG_SHELL" ]; then + "$CONFIG_SHELL" -c "./autogen.sh" +else + ./autogen.sh +fi + +mkdir -p "${BASE_BUILD_DIR}" +cd "${BASE_BUILD_DIR}" + +bash -c "${BASE_ROOT_DIR}/configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false) + +make distdir VERSION="$HOST" + +cd "${BASE_BUILD_DIR}/bitcoin-$HOST" + +bash -c "./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false) + +if [[ "${RUN_TIDY}" == "true" ]]; then + MAYBE_BEAR="bear --config src/.bear-tidy-config" + MAYBE_TOKEN="--" +fi + +bash -c "${MAYBE_BEAR} ${MAYBE_TOKEN} make $MAKEJOBS $GOAL" || ( echo "Build failure. Verbose build follows." && make "$GOAL" V=1 ; false ) + +bash -c "${PRINT_CCACHE_STATISTICS}" +du -sh "${DEPENDS_DIR}"/*/ +du -sh "${PREVIOUS_RELEASES_DIR}" + if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped - CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 - CI_EXEC make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1 - CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" + make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1 + "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped - CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 - CI_EXEC make "$MAKEJOBS" -C src minisketch/test VERBOSE=1 - CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" + make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + make "$MAKEJOBS" -C src minisketch/test VERBOSE=1 + "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" fi if [ -n "$USE_VALGRIND" ]; then - CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" + "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" fi if [ "$RUN_UNIT_TESTS" = "true" ]; then - CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" make "$MAKEJOBS" check VERBOSE=1 + bash -c "${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib make $MAKEJOBS check VERBOSE=1" fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}/bin/test_bitcoin" --catch_system_errors=no -l test_suite + bash -c "${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib ${BASE_OUTDIR}/bin/test_bitcoin --catch_system_errors=no -l test_suite" fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast + bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix ${BASE_SCRATCH_DIR}/test_runner/ --ansi --combinedlogslen=99999999 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast" fi if [ "${RUN_TIDY}" = "true" ]; then set -eo pipefail - export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/" - ( CI_EXEC run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error" - export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/" - CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\ - " src/common/args.cpp"\ - " src/common/config.cpp"\ - " src/common/init.cpp"\ - " src/common/url.cpp"\ - " src/compat"\ - " src/dbwrapper.cpp"\ - " src/init"\ - " src/kernel"\ - " src/node/chainstate.cpp"\ - " src/node/chainstatemanager_args.cpp"\ - " src/node/mempool_args.cpp"\ - " src/node/minisketchwrapper.cpp"\ - " src/node/utxo_snapshot.cpp"\ - " src/node/validation_cache_args.cpp"\ - " src/policy/feerate.cpp"\ - " src/policy/packages.cpp"\ - " src/policy/settings.cpp"\ - " src/primitives/transaction.cpp"\ - " src/random.cpp"\ - " src/rpc/fees.cpp"\ - " src/rpc/signmessage.cpp"\ - " src/test/fuzz/string.cpp"\ - " src/test/fuzz/txorphan.cpp"\ - " src/test/fuzz/util/"\ - " src/test/util/coins.cpp"\ - " src/uint256.cpp"\ - " src/util/bip32.cpp"\ - " src/util/bytevectorhash.cpp"\ - " src/util/check.cpp"\ - " src/util/error.cpp"\ - " src/util/exception.cpp"\ - " src/util/getuniquepath.cpp"\ - " src/util/hasher.cpp"\ - " src/util/message.cpp"\ - " src/util/moneystr.cpp"\ - " src/util/serfloat.cpp"\ - " src/util/spanparsing.cpp"\ - " src/util/strencodings.cpp"\ - " src/util/string.cpp"\ - " src/util/syserror.cpp"\ - " src/util/threadinterrupt.cpp"\ - " src/zmq"\ - " -p . ${MAKEJOBS}"\ - " -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp"\ - " |& tee /tmp/iwyu_ci.out" - export P_CI_DIR="${BASE_ROOT_DIR}/src" - CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/fix_includes.py --nosafe_headers < /tmp/iwyu_ci.out" - CI_EXEC "git --no-pager diff" -fi - -if [ "$RUN_SECURITY_TESTS" = "true" ]; then - CI_EXEC make test-security-check + cd "${BASE_BUILD_DIR}/bitcoin-$HOST/src/" + ( run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error" + # Filter out files by regex here, because regex may not be + # accepted in src/.bear-tidy-config + # Filter out: + # * qt qrc and moc generated files + # * secp256k1 + jq 'map(select(.file | test("src/qt/qrc_.*\\.cpp$|/moc_.*\\.cpp$|src/secp256k1/src/") | not))' ../compile_commands.json > tmp.json + mv tmp.json ../compile_commands.json + cd "${BASE_BUILD_DIR}/bitcoin-$HOST/" + python3 "${DIR_IWYU}/include-what-you-use/iwyu_tool.py" \ + -p . "${MAKEJOBS}" \ + -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" \ + -Xiwyu --max_line_length=160 \ + 2>&1 | tee /tmp/iwyu_ci.out + cd "${BASE_ROOT_DIR}/src" + python3 "${DIR_IWYU}/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out + git --no-pager diff fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then - CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}" -fi - -if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - echo "Stop and remove CI container by ID" - docker container kill "${CI_CONTAINER_ID}" + bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN}" fi diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh index 93b07aab1e..2284a2903b 100755 --- a/ci/test_run_all.sh +++ b/ci/test_run_all.sh @@ -8,6 +8,10 @@ export LC_ALL=C.UTF-8 set -o errexit; source ./ci/test/00_setup_env.sh set -o errexit; source ./ci/test/04_install.sh -set -o errexit; source ./ci/test/05_before_script.sh -set -o errexit; source ./ci/test/06_script_a.sh -set -o errexit; source ./ci/test/06_script_b.sh +set -o errexit +CI_EXEC "${BASE_ROOT_DIR}/ci/test/06_script_b.sh" + +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then + echo "Stop and remove CI container by ID" + docker container kill "${CI_CONTAINER_ID}" +fi diff --git a/configure.ac b/configure.ac index cbbf8d6172..8015813ec7 100644 --- a/configure.ac +++ b/configure.ac @@ -70,11 +70,12 @@ else fi AC_PROG_CXX -dnl By default, libtool for mingw refuses to link static libs into a dll for -dnl fear of mixing pic/non-pic objects, and import/export complications. Since -dnl we have those under control, re-enable that functionality. +dnl libtool overrides case $host in *mingw*) + dnl By default, libtool for mingw refuses to link static libs into a dll for + dnl fear of mixing pic/non-pic objects, and import/export complications. Since + dnl we have those under control, re-enable that functionality. lt_cv_deplibs_check_method="pass_all" dnl Remove unwanted -DDLL_EXPORT from these variables. @@ -83,6 +84,16 @@ case $host in lt_cv_prog_compiler_pic="-DPIC" lt_cv_prog_compiler_pic_CXX="-DPIC" ;; + *darwin*) + dnl Because it prints a verbose warning, lld fails the following check + dnl for "-Wl,-single_module" from libtool.m4: + dnl # If there is a non-empty error log, and "single_module" + dnl # appears in it, assume the flag caused a linker warning + dnl "-single_module" works fine on ld64 and lld, so just bypass the test. + dnl Failure to set this to "yes" causes libtool to use a very broken + dnl link-line for shared libs. + lt_cv_apple_cc_single_mod="yes" + ;; esac AC_ARG_WITH([seccomp], @@ -104,9 +115,6 @@ else AX_CXX_COMPILE_STDCXX([20], [noext], [mandatory]) fi -dnl check if additional link flags are required for std::filesystem -CHECK_FILESYSTEM - dnl Unless the user specified OBJCXX, force it to be the same as CXX. This ensures dnl that we get the same -std flags for both. m4_ifdef([AC_PROG_OBJCXX],[ @@ -1064,7 +1072,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]], dnl Check for posix_fallocate AC_MSG_CHECKING([for posix_fallocate]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - // same as in src/util/system.cpp + // same as in src/util/fs_helpers.cpp #ifdef __linux__ #ifdef _POSIX_C_SOURCE #undef _POSIX_C_SOURCE @@ -1162,17 +1170,16 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ctime>]], ) dnl Check for different ways of gathering OS randomness -AC_MSG_CHECKING([for Linux getrandom syscall]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> - #include <sys/syscall.h> - #include <linux/random.h>]], - [[ syscall(SYS_getrandom, nullptr, 32, 0); ]])], - [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYS_GETRANDOM], [1], [Define this symbol if the Linux getrandom system call is available]) ], +AC_MSG_CHECKING([for Linux getrandom function]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include <sys/random.h>]], + [[ getrandom(nullptr, 32, 0); ]])], + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GETRANDOM], [1], [Define this symbol if the Linux getrandom function call is available]) ], [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING([for getentropy via random.h]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> +AC_MSG_CHECKING([for getentropy via sys/random.h]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <sys/random.h>]], [[ getentropy(nullptr, 32) ]])], [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GETENTROPY_RAND], [1], [Define this symbol if the BSD getentropy system call is available with sys/random.h]) ], @@ -1401,7 +1408,9 @@ if test "$use_usdt" != "no"; then AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [#include <sys/sdt.h>], - [DTRACE_PROBE("context", "event");] + [DTRACE_PROBE(context, event); + int a, b, c, d, e, f, g; + DTRACE_PROBE7(context, event, a, b, c, d, e, f, g);] )], [AC_MSG_RESULT([yes]); AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable tracepoints for Userspace, Statically Defined Tracing])], [AC_MSG_RESULT([no]); use_usdt=no;] @@ -1482,10 +1491,6 @@ if test "$use_boost" = "yes"; then AX_CHECK_PREPROC_FLAG([-DBOOST_NO_CXX98_FUNCTION_BASE], [BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_CXX98_FUNCTION_BASE"], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([[#include <boost/config.hpp>]])]) - if test "$enable_debug" = "yes" || test "$enable_fuzz" = "yes"; then - BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE" - fi - if test "$suppress_external_warnings" != "no"; then BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) fi @@ -1622,12 +1627,9 @@ dnl ZMQ check if test "$use_zmq" = "yes"; then PKG_CHECK_MODULES([ZMQ], [libzmq >= 4], - AC_DEFINE([ENABLE_ZMQ], [1], [Define to 1 to enable ZMQ functions]), - [AC_DEFINE([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) - AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) + AC_DEFINE([ENABLE_ZMQ], [1], [Define this symbol to enable ZMQ functions]), + [AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) use_zmq=no]) -else - AC_DEFINE_UNQUOTED([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) fi if test "$use_zmq" = "yes"; then @@ -1639,6 +1641,8 @@ if test "$use_zmq" = "yes"; then esac fi +AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) + dnl libmultiprocess library check libmultiprocess_found=no @@ -1833,8 +1837,6 @@ if test "$bitcoin_enable_qt" != "no"; then fi fi -AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) - AC_MSG_CHECKING([whether to build test_bitcoin]) if test "$use_tests" = "yes"; then if test "$enable_fuzz" = "yes"; then diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 6cd022ef17..452a1d42d6 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -10,7 +10,7 @@ Otherwise the exit status will be 1 and it will log which executables failed whi import sys from typing import List -import lief #type:ignore +import lief def check_ELF_RELRO(binary) -> bool: ''' diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index f26236dd59..4fb997b023 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -13,7 +13,7 @@ Example usage: import sys from typing import List, Dict -import lief #type:ignore +import lief # Debian 10 (Buster) EOL: 2024. https://wiki.debian.org/LTS # @@ -72,6 +72,25 @@ ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { }, } +ELF_ABIS: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, List[int]]] = { + lief.ELF.ARCH.x86_64: { + lief.ENDIANNESS.LITTLE: [3,2,0], + }, + lief.ELF.ARCH.ARM: { + lief.ENDIANNESS.LITTLE: [3,2,0], + }, + lief.ELF.ARCH.AARCH64: { + lief.ENDIANNESS.LITTLE: [3,7,0], + }, + lief.ELF.ARCH.PPC64: { + lief.ENDIANNESS.LITTLE: [3,10,0], + lief.ENDIANNESS.BIG: [3,2,0], + }, + lief.ELF.ARCH.RISCV: { + lief.ENDIANNESS.LITTLE: [4,15,0], + }, +} + # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt @@ -242,12 +261,19 @@ def check_ELF_interpreter(binary) -> bool: return binary.concrete.interpreter == expected_interpreter +def check_ELF_ABI(binary) -> bool: + expected_abi = ELF_ABIS[binary.header.machine_type][binary.abstract.header.endianness] + note = binary.concrete.get(lief.ELF.NOTE_TYPES.ABI_TAG) + assert note.details.abi == lief.ELF.NOTE_ABIS.LINUX + return note.details.version == expected_abi + CHECKS = { lief.EXE_FORMATS.ELF: [ ('IMPORTED_SYMBOLS', check_imported_symbols), ('EXPORTED_SYMBOLS', check_exported_symbols), ('LIBRARY_DEPENDENCIES', check_ELF_libraries), ('INTERPRETER_NAME', check_ELF_interpreter), + ('ABI', check_ELF_ABI), ], lief.EXE_FORMATS.MACHO: [ ('DYNAMIC_LIBRARIES', check_MACHO_libraries), diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 54718fd7a1..d666291cba 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -5,7 +5,7 @@ ''' Test script for security-check.py ''' -import lief #type:ignore +import lief import os import subprocess from typing import List diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index d83ff08713..f1c2854d09 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -11,7 +11,6 @@ (gnu packages commencement) (gnu packages compression) (gnu packages cross-base) - (gnu packages curl) (gnu packages file) (gnu packages gawk) (gnu packages gcc) @@ -257,9 +256,6 @@ and abstract ELF, PE and MachO formats.") (build-system cmake-build-system) (inputs `(("openssl", openssl))) - (arguments - '(#:configure-flags - (list "-DCMAKE_DISABLE_FIND_PACKAGE_CURL=TRUE"))) (home-page "https://github.com/mtrojnar/osslsigncode") (synopsis "Authenticode signing and timestamping tool") (description "osslsigncode is a small tool that implements part of the @@ -559,9 +555,7 @@ inspecting signatures in Mach-O binaries.") (sha256 (base32 "0azpb9cvnbv25zg8019rqz48h8i2257ngyjg566dlnp74ivrs9vq")) - (patches (search-our-patches "glibc-ldd-x86_64.patch" - "glibc-versioned-locpath.patch" - "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" + (patches (search-our-patches "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" "glibc-2.27-fcommon.patch" "glibc-2.27-guix-prefix.patch")))))) diff --git a/contrib/guix/patches/gcc-broken-longjmp.patch b/contrib/guix/patches/gcc-broken-longjmp.patch index 1cfc0918b0..56568813c0 100644 --- a/contrib/guix/patches/gcc-broken-longjmp.patch +++ b/contrib/guix/patches/gcc-broken-longjmp.patch @@ -29,6 +29,8 @@ Date: Wed May 5 22:48:51 2021 +0200 gcc/testsuite/ * gcc.c-torture/execute/20210505-1.c: New test. + This patch can be dropped when we are building with GCC 10.4.0 or later. + diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c index 2f838840e96..06ad1b2274e 100644 --- a/gcc/config/i386/i386.c diff --git a/contrib/guix/patches/glibc-2.27-fcommon.patch b/contrib/guix/patches/glibc-2.27-fcommon.patch index f3baacab98..817aa85bb9 100644 --- a/contrib/guix/patches/glibc-2.27-fcommon.patch +++ b/contrib/guix/patches/glibc-2.27-fcommon.patch @@ -5,7 +5,7 @@ Date: Fri May 6 11:03:04 2022 +0100 build: use -fcommon to retain legacy behaviour with GCC 10 GCC 10 started using -fno-common by default, which causes issues with - the powerpc builds using gibc 2.24. A patch was commited to glibc to fix + the powerpc builds using gibc 2.27. A patch was commited to glibc to fix the issue, 18363b4f010da9ba459b13310b113ac0647c2fcc but is non-trvial to backport, and was broken in at least one way, see the followup in commit 7650321ce037302bfc2f026aa19e0213b8d02fe6. @@ -17,6 +17,8 @@ Date: Fri May 6 11:03:04 2022 +0100 https://sourceware.org/git/?p=glibc.git;a=commit;h=18363b4f010da9ba459b13310b113ac0647c2fcc https://sourceware.org/git/?p=glibc.git;a=commit;h=7650321ce037302bfc2f026aa19e0213b8d02fe6 + This patch can be dropped when we are building with glibc 2.31+. + diff --git a/Makeconfig b/Makeconfig index 86a71e5802..aa2166be60 100644 --- a/Makeconfig diff --git a/contrib/guix/patches/glibc-2.27-guix-prefix.patch b/contrib/guix/patches/glibc-2.27-guix-prefix.patch index 6648bc6c05..dc515907ff 100644 --- a/contrib/guix/patches/glibc-2.27-guix-prefix.patch +++ b/contrib/guix/patches/glibc-2.27-guix-prefix.patch @@ -5,7 +5,7 @@ In order to be reproducible regardless of the architecture used to build the package, map all guix store prefixes to something fixed, e.g. /usr. We might be able to drop this in favour of using --with-nonshared-cflags -when we being using newer versions of glibc. +when we begin using newer versions of glibc. --- a/Makeconfig +++ b/Makeconfig diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch index c0f8495c41..ab8ae9c023 100644 --- a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch @@ -4,6 +4,8 @@ See also: http://lists.busybox.net/pipermail/buildroot/2020-July/590376.html. https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e +This patch can be dropped when we are building with glibc 2.28+. + From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 From: "H.J. Lu" <hjl.tools@gmail.com> Date: Mon, 31 Dec 2018 09:26:42 -0800 diff --git a/contrib/guix/patches/glibc-ldd-x86_64.patch b/contrib/guix/patches/glibc-ldd-x86_64.patch deleted file mode 100644 index a23b095caa..0000000000 --- a/contrib/guix/patches/glibc-ldd-x86_64.patch +++ /dev/null @@ -1,10 +0,0 @@ -By default, 'RTDLLIST' in 'ldd' refers to 'lib64/ld-linux-x86-64.so', whereas -it's in 'lib/' for us. This patch fixes that. - ---- a/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed -+++ b/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed -@@ -1,3 +1,3 @@ - /LD_TRACE_LOADED_OBJECTS=1/a\ - add_env="$add_env LD_LIBRARY_VERSION=\\$verify_out" --s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \264\4-x86-64\6 \2x32\4-x32\6"_ -+s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \2\4-x86-64\6 \2x32\4-x32\6"_ diff --git a/contrib/guix/patches/glibc-versioned-locpath.patch b/contrib/guix/patches/glibc-versioned-locpath.patch deleted file mode 100644 index bc7652127f..0000000000 --- a/contrib/guix/patches/glibc-versioned-locpath.patch +++ /dev/null @@ -1,240 +0,0 @@ -The format of locale data can be incompatible between libc versions, and -loading incompatible data can lead to 'setlocale' returning EINVAL at best -or triggering an assertion failure at worst. See -https://lists.gnu.org/archive/html/guix-devel/2015-09/msg00717.html -for background information. - -To address that, this patch changes libc to honor a new 'GUIX_LOCPATH' -variable, and to look for locale data in version-specific sub-directories of -that variable. So, if GUIX_LOCPATH=/foo:/bar, locale data is searched for in -/foo/X.Y and /bar/X.Y, where X.Y is the libc version number. - -That way, a single 'GUIX_LOCPATH' setting can work even if different libc -versions coexist on the system. - ---- a/locale/newlocale.c -+++ b/locale/newlocale.c -@@ -30,6 +30,7 @@ - /* Lock for protecting global data. */ - __libc_rwlock_define (extern , __libc_setlocale_lock attribute_hidden) - -+extern error_t compute_locale_search_path (char **, size_t *); - - /* Use this when we come along an error. */ - #define ERROR_RETURN \ -@@ -48,7 +49,6 @@ __newlocale (int category_mask, const char *locale, __locale_t base) - __locale_t result_ptr; - char *locale_path; - size_t locale_path_len; -- const char *locpath_var; - int cnt; - size_t names_len; - -@@ -102,17 +102,8 @@ __newlocale (int category_mask, const char *locale, __locale_t base) - locale_path = NULL; - locale_path_len = 0; - -- locpath_var = getenv ("LOCPATH"); -- if (locpath_var != NULL && locpath_var[0] != '\0') -- { -- if (__argz_create_sep (locpath_var, ':', -- &locale_path, &locale_path_len) != 0) -- return NULL; -- -- if (__argz_add_sep (&locale_path, &locale_path_len, -- _nl_default_locale_path, ':') != 0) -- return NULL; -- } -+ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) -+ return NULL; - - /* Get the names for the locales we are interested in. We either - allow a composite name or a single name. */ -diff --git a/locale/setlocale.c b/locale/setlocale.c -index ead030d..0c0e314 100644 ---- a/locale/setlocale.c -+++ b/locale/setlocale.c -@@ -215,12 +215,65 @@ setdata (int category, struct __locale_data *data) - } - } - -+/* Return in *LOCALE_PATH and *LOCALE_PATH_LEN the locale data search path as -+ a colon-separated list. Return ENOMEN on error, zero otherwise. */ -+error_t -+compute_locale_search_path (char **locale_path, size_t *locale_path_len) -+{ -+ char* guix_locpath_var = getenv ("GUIX_LOCPATH"); -+ char *locpath_var = getenv ("LOCPATH"); -+ -+ if (guix_locpath_var != NULL && guix_locpath_var[0] != '\0') -+ { -+ /* Entries in 'GUIX_LOCPATH' take precedence over 'LOCPATH'. These -+ entries are systematically prefixed with "/X.Y" where "X.Y" is the -+ libc version. */ -+ if (__argz_create_sep (guix_locpath_var, ':', -+ locale_path, locale_path_len) != 0 -+ || __argz_suffix_entries (locale_path, locale_path_len, -+ "/" VERSION) != 0) -+ goto bail_out; -+ } -+ -+ if (locpath_var != NULL && locpath_var[0] != '\0') -+ { -+ char *reg_locale_path = NULL; -+ size_t reg_locale_path_len = 0; -+ -+ if (__argz_create_sep (locpath_var, ':', -+ ®_locale_path, ®_locale_path_len) != 0) -+ goto bail_out; -+ -+ if (__argz_append (locale_path, locale_path_len, -+ reg_locale_path, reg_locale_path_len) != 0) -+ goto bail_out; -+ -+ free (reg_locale_path); -+ } -+ -+ if (*locale_path != NULL) -+ { -+ /* Append the system default locale directory. */ -+ if (__argz_add_sep (locale_path, locale_path_len, -+ _nl_default_locale_path, ':') != 0) -+ goto bail_out; -+ } -+ -+ return 0; -+ -+ bail_out: -+ free (*locale_path); -+ *locale_path = NULL; -+ *locale_path_len = 0; -+ -+ return ENOMEM; -+} -+ - char * - setlocale (int category, const char *locale) - { - char *locale_path; - size_t locale_path_len; -- const char *locpath_var; - char *composite; - - /* Sanity check for CATEGORY argument. */ -@@ -251,17 +304,10 @@ setlocale (int category, const char *locale) - locale_path = NULL; - locale_path_len = 0; - -- locpath_var = getenv ("LOCPATH"); -- if (locpath_var != NULL && locpath_var[0] != '\0') -+ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) - { -- if (__argz_create_sep (locpath_var, ':', -- &locale_path, &locale_path_len) != 0 -- || __argz_add_sep (&locale_path, &locale_path_len, -- _nl_default_locale_path, ':') != 0) -- { -- __libc_rwlock_unlock (__libc_setlocale_lock); -- return NULL; -- } -+ __libc_rwlock_unlock (__libc_setlocale_lock); -+ return NULL; - } - - if (category == LC_ALL) -diff --git a/string/Makefile b/string/Makefile -index 8424a61..f925503 100644 ---- a/string/Makefile -+++ b/string/Makefile -@@ -38,7 +38,7 @@ routines := strcat strchr strcmp strcoll strcpy strcspn \ - swab strfry memfrob memmem rawmemchr strchrnul \ - $(addprefix argz-,append count create ctsep next \ - delete extract insert stringify \ -- addsep replace) \ -+ addsep replace suffix) \ - envz basename \ - strcoll_l strxfrm_l string-inlines memrchr \ - xpg-strerror strerror_l -diff --git a/string/argz-suffix.c b/string/argz-suffix.c -new file mode 100644 -index 0000000..505b0f2 ---- /dev/null -+++ b/string/argz-suffix.c -@@ -0,0 +1,56 @@ -+/* Copyright (C) 2015 Free Software Foundation, Inc. -+ This file is part of the GNU C Library. -+ Contributed by Ludovic Courtès <ludo@gnu.org>. -+ -+ The GNU C Library is free software; you can redistribute it and/or -+ modify it under the terms of the GNU Lesser General Public -+ License as published by the Free Software Foundation; either -+ version 2.1 of the License, or (at your option) any later version. -+ -+ The GNU C Library is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+ Lesser General Public License for more details. -+ -+ You should have received a copy of the GNU Lesser General Public -+ License along with the GNU C Library; if not, see -+ <http://www.gnu.org/licenses/>. */ -+ -+#include <argz.h> -+#include <errno.h> -+#include <stdlib.h> -+#include <string.h> -+ -+ -+error_t -+__argz_suffix_entries (char **argz, size_t *argz_len, const char *suffix) -+ -+{ -+ size_t suffix_len = strlen (suffix); -+ size_t count = __argz_count (*argz, *argz_len); -+ size_t new_argz_len = *argz_len + count * suffix_len; -+ char *new_argz = malloc (new_argz_len); -+ -+ if (new_argz) -+ { -+ char *p = new_argz, *entry; -+ -+ for (entry = *argz; -+ entry != NULL; -+ entry = argz_next (*argz, *argz_len, entry)) -+ { -+ p = stpcpy (p, entry); -+ p = stpcpy (p, suffix); -+ p++; -+ } -+ -+ free (*argz); -+ *argz = new_argz; -+ *argz_len = new_argz_len; -+ -+ return 0; -+ } -+ else -+ return ENOMEM; -+} -+weak_alias (__argz_suffix_entries, argz_suffix_entries) -diff --git a/string/argz.h b/string/argz.h -index bb62a31..d276a35 100644 ---- a/string/argz.h -+++ b/string/argz.h -@@ -134,6 +134,16 @@ extern error_t argz_replace (char **__restrict __argz, - const char *__restrict __str, - const char *__restrict __with, - unsigned int *__restrict __replace_count); -+ -+/* Suffix each entry of ARGZ & ARGZ_LEN with SUFFIX. Return 0 on success, -+ and ENOMEN if memory cannot be allocated. */ -+extern error_t __argz_suffix_entries (char **__restrict __argz, -+ size_t *__restrict __argz_len, -+ const char *__restrict __suffix); -+extern error_t argz_suffix_entries (char **__restrict __argz, -+ size_t *__restrict __argz_len, -+ const char *__restrict __suffix); -+ - - /* Returns the next entry in ARGZ & ARGZ_LEN after ENTRY, or NULL if there - are no more. If entry is NULL, then the first entry is returned. This diff --git a/contrib/guix/patches/vmov-alignment.patch b/contrib/guix/patches/vmov-alignment.patch index 072f76eafd..7976b864af 100644 --- a/contrib/guix/patches/vmov-alignment.patch +++ b/contrib/guix/patches/vmov-alignment.patch @@ -1,6 +1,7 @@ Description: Use unaligned VMOV instructions Author: Stephen Kitt <skitt@debian.org> Bug-Debian: https://bugs.debian.org/939559 +See also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412 Based on a patch originally by Claude Heiland-Allen <claude@mathr.co.uk> diff --git a/contrib/init/bitcoind.service b/contrib/init/bitcoind.service index 93de353bb4..87da17f955 100644 --- a/contrib/init/bitcoind.service +++ b/contrib/init/bitcoind.service @@ -18,10 +18,11 @@ After=network-online.target Wants=network-online.target [Service] -ExecStart=/usr/bin/bitcoind -daemonwait \ - -pid=/run/bitcoind/bitcoind.pid \ +ExecStart=/usr/bin/bitcoind -pid=/run/bitcoind/bitcoind.pid \ -conf=/etc/bitcoin/bitcoin.conf \ - -datadir=/var/lib/bitcoind + -datadir=/var/lib/bitcoind \ + -startupnotify='systemd-notify --ready' \ + -shutdownnotify='systemd-notify --stopping' # Make sure the config directory is readable by the service user PermissionsStartOnly=true @@ -30,8 +31,10 @@ ExecStartPre=/bin/chgrp bitcoin /etc/bitcoin # Process management #################### -Type=forking +Type=notify +NotifyAccess=all PIDFile=/run/bitcoind/bitcoind.pid + Restart=on-failure TimeoutStartSec=infinity TimeoutStopSec=600 diff --git a/contrib/tracing/mempool_monitor.py b/contrib/tracing/mempool_monitor.py index 9d427d4632..afb5e60372 100755 --- a/contrib/tracing/mempool_monitor.py +++ b/contrib/tracing/mempool_monitor.py @@ -27,7 +27,7 @@ PROGRAM = """ struct added_event { u8 hash[HASH_LENGTH]; - u64 vsize; + s32 vsize; s64 fee; }; @@ -35,7 +35,7 @@ struct removed_event { u8 hash[HASH_LENGTH]; char reason[MAX_REMOVAL_REASON_LENGTH]; - u64 vsize; + s32 vsize; s64 fee; u64 entry_time; }; @@ -49,11 +49,11 @@ struct rejected_event struct replaced_event { u8 replaced_hash[HASH_LENGTH]; - u64 replaced_vsize; + s32 replaced_vsize; s64 replaced_fee; u64 replaced_entry_time; u8 replacement_hash[HASH_LENGTH]; - u64 replacement_vsize; + s32 replacement_vsize; s64 replacement_fee; }; diff --git a/contrib/verify-binaries/README.md b/contrib/verify-binaries/README.md index c62d760e1a..04d683e69b 100644 --- a/contrib/verify-binaries/README.md +++ b/contrib/verify-binaries/README.md @@ -17,7 +17,7 @@ must obtain that key for your local GPG installation. You can obtain these keys by - through a browser using a key server (e.g. keyserver.ubuntu.com), - manually using the `gpg --keyserver <url> --recv-keys <key>` command, or - - you can run the packaged `verify.py ... --import-keys` script to + - you can run the packaged `verify.py --import-keys ...` script to have it automatically retrieve unrecognized keys. #### Usage diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index 94daf28b15..f25486776f 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -2,3 +2,4 @@ E777299FC265DD04793070EB944D35F9AC3DB76A D1DBF2C4B96F2DEBF4C16654410108112E7EA81F 152812300785C96444D3334D17565732E08E5E41 6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C +4D1B3D5ECBA1A7E05371EEBE46800E30FC748A66 diff --git a/depends/README.md b/depends/README.md index 1064b7d18a..8bf751ab30 100644 --- a/depends/README.md +++ b/depends/README.md @@ -15,6 +15,7 @@ For example: **Bitcoin Core's `configure` script by default will ignore the depends output.** In order for it to pick up libraries, tools, and settings from the depends build, you must set the `CONFIG_SITE` environment variable to point to a `config.site` settings file. +Make sure that `CONFIG_SITE` is an absolute path. In the above example, a file named `depends/x86_64-w64-mingw32/share/config.site` will be created. To use it during compilation: diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 522a6b17ef..08d81cbc99 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -63,7 +63,7 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$( # Explicitly point to our binaries (e.g. cctools) so that they are # ensured to be found and preferred over other possibilities. # -# -stdlib=libc++ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 +# -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 # # Forces clang to use the libc++ headers from our SDK and completely # forget about the libc++ headers from the standard directories @@ -107,7 +107,6 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ -isysroot$(OSX_SDK) \ - -stdlib=libc++ \ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \ -Xclang -internal-externc-isystem -Xclang $(clang_resource_dir)/include \ -Xclang -internal-externc-isystem -Xclang $(OSX_SDK)/usr/include diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk index f2712294ab..b11037b83e 100644 --- a/depends/packages/native_clang.mk +++ b/depends/packages/native_clang.mk @@ -9,10 +9,6 @@ $(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16 $(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231 endif -define $(package)_preprocess_cmds - rm -f $($(package)_extract_dir)/lib/libc++abi.so* -endef - define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin && \ diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index ab5db58cdd..6cbb6ebd72 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -5,6 +5,41 @@ The headless daemon `bitcoind` has the JSON-RPC API enabled by default, the GUI option. In the GUI it is possible to execute RPC methods in the Debug Console Dialog. +## Endpoints + +There are two JSON-RPC endpoints on the server: + +1. `/` +2. `/wallet/<walletname>/` + +### `/` endpoint + +This endpoint is always active. +It can always service non-wallet requests and can service wallet requests when +exactly one wallet is loaded. + +### `/wallet/<walletname>/` endpoint + +This endpoint is only activated when the wallet component has been compiled in. +It can service both wallet and non-wallet requests. +It MUST be used for wallet requests when two or more wallets are loaded. + +This is the endpoint used by bitcoin-cli when a `-rpcwallet=` parameter is passed in. + +Best practice would dictate using the `/wallet/<walletname>/` endpoint for ALL +requests when multiple wallets are in use. + +### Examples + +```sh +# Get block count from the / endpoint when rpcuser=alice and rpcport=38332 +$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getblockcount", "params": []}' -H 'content-type: text/plain;' localhost:38332/ + +# Get balance from the /wallet/walletname endpoint when rpcuser=alice, rpcport=38332 and rpcwallet=desc-wallet +$ curl --user alice --data-binary '{"jsonrpc": "1.0", "id": "0", "method": "getbalance", "params": []}' -H 'content-type: text/plain;' localhost:38332/wallet/desc-wallet + +``` + ## Parameter passing The JSON-RPC server supports both _by-position_ and _by-name_ [parameter diff --git a/doc/bips.md b/doc/bips.md index 1d5c91b8bd..94213f8048 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -1,4 +1,4 @@ -BIPs that are implemented by Bitcoin Core (up-to-date up to **v24.0**): +BIPs that are implemented by Bitcoin Core: * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1** ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575)) * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). diff --git a/doc/build-unix.md b/doc/build-unix.md index 0960ae1577..3633d4f811 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -4,16 +4,6 @@ Some notes on how to build Bitcoin Core in Unix. (For BSD specific instructions, see `build-*bsd.md` in this directory.) -Note ---------------------- -Always use absolute paths to configure and compile Bitcoin Core and the dependencies. -For example, when specifying the path of the dependency: - - ../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX - -Here BDB_PREFIX must be an absolute path - it is defined using $(pwd) which ensures -the usage of the absolute path. - To Build --------------------- @@ -24,12 +14,11 @@ make # use "-j N" for N parallel jobs make install # optional ``` -This will build bitcoin-qt as well, if the dependencies are met. - -See [dependencies.md](dependencies.md) for a complete overview. +See below for instructions on how to [install the dependencies on popular Linux +distributions](#linux-distribution-specific-instructions), or the +[dependencies](#dependencies) section for a complete overview. -Memory Requirements --------------------- +## Memory Requirements C++ compilers are memory-hungry. It is recommended to have at least 1.5 GB of memory available when compiling Bitcoin Core. On systems with less, gcc can be @@ -57,7 +46,7 @@ Build requirements: sudo apt-get install build-essential libtool autotools-dev automake pkg-config bsdmainutils python3 -Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: +Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: sudo apt-get install libevent-dev libboost-dev @@ -65,7 +54,7 @@ SQLite is required for the descriptor wallet: sudo apt install libsqlite3-dev -Berkeley DB is required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, +Berkeley DB is only required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, but these will install Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). @@ -114,7 +103,7 @@ Build requirements: sudo dnf install gcc-c++ libtool make autoconf automake python3 -Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: +Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: sudo dnf install libevent-devel boost-devel @@ -126,7 +115,7 @@ Berkeley DB is required for the legacy wallet: sudo dnf install libdb4-devel libdb4-cxx-devel -Newer Fedora releases, since Fedora 33, have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install +Berkeley DB is only required for the legacy wallet. Newer Fedora releases have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on Berkeley DB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). @@ -166,27 +155,13 @@ libqrencode (optional) can be installed with: Once these are installed, they will be found by configure and a bitcoin-qt executable will be built by default. -Notes ------ -The release is built with GCC and then "strip bitcoind" to strip the debug -symbols, which reduces the executable size by about 90%. +## Dependencies -miniupnpc ---------- +See [dependencies.md](dependencies.md) for a complete overview, and +[depends](/depends/README.md) on how to compile them yourself, if you wish to +not use the packages of your Linux distribution. -[miniupnpc](https://miniupnp.tuxfamily.org) may be used for UPnP port mapping. It can be downloaded from [here]( -https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and -turned off by default. - -libnatpmp ---------- - -[libnatpmp](https://miniupnp.tuxfamily.org/libnatpmp.html) may be used for NAT-PMP port mapping. It can be downloaded -from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and -turned off by default. - -Berkeley DB ------------ +### Berkeley DB The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is recommended to use Berkeley DB 4.8. If you have to build it yourself, and don't @@ -205,53 +180,9 @@ export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-pc-linux-gnu" BDB_CFLAGS="-I${BDB_PREFIX}/include" ``` -**Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). - -Security --------- -To help make your Bitcoin Core installation more secure by making certain attacks impossible to -exploit even if a vulnerability is found, binaries are hardened by default. -This can be disabled with: - -Hardening Flags: +**Note**: Make sure that `BDB_PREFIX` is an absolute path. - ./configure --enable-hardening - ./configure --disable-hardening - - -Hardening enables the following features: -* _Position Independent Executable_: Build position independent code to take advantage of Address Space Layout Randomization - offered by some kernels. Attackers who can cause execution of code at an arbitrary memory - location are thwarted if they don't know where anything useful is located. - The stack and heap are randomly located by default, but this allows the code section to be - randomly located as well. - - On an AMD64 processor where a library was not compiled with -fPIC, this will cause an error - such as: "relocation R_X86_64_32 against `......' can not be used when making a shared object;" - - To test that you have built PIE executable, install scanelf, part of paxutils, and use: - - scanelf -e ./bitcoin - - The output should contain: - - TYPE - ET_DYN - -* _Non-executable Stack_: If the stack is executable then trivial stack-based buffer overflow exploits are possible if - vulnerable buffers are found. By default, Bitcoin Core should be built with a non-executable stack, - but if one of the libraries it uses asks for an executable stack or someone makes a mistake - and uses a compiler extension which requires an executable stack, it will silently build an - executable without the non-executable stack protection. - - To verify that the stack is non-executable after compiling use: - `scanelf -e ./bitcoin` - - The output should contain: - STK/REL/PTL - RW- R-- RW- - - The STK RW- means that the stack is readable and writeable but not executable. +**Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). Disable-wallet mode -------------------- diff --git a/doc/cjdns.md b/doc/cjdns.md index b69564729f..031cd1978b 100644 --- a/doc/cjdns.md +++ b/doc/cjdns.md @@ -112,5 +112,4 @@ There are several ways to see your CJDNS address in Bitcoin Core: To see which CJDNS peers your node is connected to, use `bitcoin-cli -netinfo 4` or the `getpeerinfo` RPC (i.e. `bitcoin-cli getpeerinfo`). -To see which CJDNS addresses your node knows, use the `getnodeaddresses 0 cjdns` -RPC. +You can use the `getnodeaddresses` RPC to fetch a number of CJDNS peers known to your node; run `bitcoin-cli help getnodeaddresses` for details. diff --git a/doc/dependencies.md b/doc/dependencies.md index c8417f49e6..804f796abe 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -8,8 +8,8 @@ You can find installation instructions in the `build-*.md` file for your platfor | --- | --- | | [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) | | [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | -| [Clang](https://clang.llvm.org) | [8.0](https://github.com/bitcoin/bitcoin/pull/24164) | -| [GCC](https://gcc.gnu.org) | [8.1](https://github.com/bitcoin/bitcoin/pull/23060) | +| [Clang](https://clang.llvm.org) | [10.0](https://github.com/bitcoin/bitcoin/pull/27682) | +| [GCC](https://gcc.gnu.org) | [9.1](https://github.com/bitcoin/bitcoin/pull/27662) | | [Python](https://www.python.org) (scripts, tests) | [3.8](https://github.com/bitcoin/bitcoin/pull/27483) | | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | @@ -20,7 +20,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.81.0](https://github.com/bitcoin/bitcoin/pull/26557) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | [libevent](../depends/packages/libevent.mk) | [link](https://github.com/libevent/libevent/releases) | [2.1.12-stable](https://github.com/bitcoin/bitcoin/pull/21991) | [2.1.8](https://github.com/bitcoin/bitcoin/pull/24681) | No | | glibc | [link](https://www.gnu.org/software/libc/) | N/A | [2.27](https://github.com/bitcoin/bitcoin/pull/27029) | Yes | -| Linux Kernel | [link](https://www.kernel.org/) | N/A | 3.2.0 | Yes | +| Linux Kernel | [link](https://www.kernel.org/) | N/A | [3.17.0](https://github.com/bitcoin/bitcoin/pull/27699) | Yes | ## Optional diff --git a/doc/i2p.md b/doc/i2p.md index 0432136554..cfff426617 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -109,8 +109,7 @@ incoming I2P connections (`-i2pacceptincoming`): To see which I2P peers your node is connected to, use `bitcoin-cli -netinfo 4` or the `getpeerinfo` RPC (e.g. `bitcoin-cli getpeerinfo`). -To see which I2P addresses your node knows, use the `getnodeaddresses 0 i2p` -RPC. +You can use the `getnodeaddresses` RPC to fetch a number of I2P peers known to your node; run `bitcoin-cli help getnodeaddresses` for details. ## Compatibility diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index 25205258b8..dbe88d45f3 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -43,7 +43,7 @@ threads take up 8MiB for the thread stack on a 64-bit system, and 4MiB in a ## Linux specific -By default, since glibc `2.10`, the C library will create up to two heap arenas per core. This is known to cause excessive memory usage in some scenarios. To avoid this make a script that sets `MALLOC_ARENA_MAX` before starting bitcoind: +By default, glibc will create up to two heap arenas per core. This is known to cause excessive memory usage in some scenarios. To avoid this make a script that sets `MALLOC_ARENA_MAX` before starting bitcoind: ```bash #!/usr/bin/env bash diff --git a/doc/release-notes-26076.md b/doc/release-notes-26076.md new file mode 100644 index 0000000000..f95e4be0e3 --- /dev/null +++ b/doc/release-notes-26076.md @@ -0,0 +1,13 @@ +RPC +--- + +- The `listdescriptors`, `decodepsbt` and similar RPC methods now show `h` rather than apostrophe (`'`) to indicate + hardened derivation. This does not apply when using the `private` parameter, which + matches the marker used when descriptor was generated or imported. Newly created + wallets use `h`. This change makes it easier to handle descriptor strings manually. + E.g. the `importdescriptors` RPC call is easiest to use `h` as the marker: `'["desc": ".../0h/..."]'`. + With this change `listdescriptors` will use `h`, so you can copy-paste the result, + without having to add escape characters or switch `'` to 'h' manually. + Note that this changes the descriptor checksum. + For legacy wallets the `hdkeypath` field in `getaddressinfo` is unchanged, + nor is the serialization format of wallet dumps. (#26076) diff --git a/doc/release-notes-26485.md b/doc/release-notes-26485.md new file mode 100644 index 0000000000..c8df3d22fb --- /dev/null +++ b/doc/release-notes-26485.md @@ -0,0 +1,16 @@ +JSON-RPC +--- + +For RPC methods which accept `options` parameters ((`importmulti`, `listunspent`, `fundrawtransaction`, `bumpfee`, `send`, `sendall`, `walletcreatefundedpsbt`, `simulaterawtransaction`), it is now possible to pass the options as named parameters without the need for a nested object. (#26485) + +This means it is possible make calls like: + +```sh +src/bitcoin-cli -named bumpfee txid fee_rate=100 +``` + +instead of + +```sh +src/bitcoin-cli -named bumpfee txid options='{"fee_rate": 100}' +``` diff --git a/doc/release-notes-27302.md b/doc/release-notes-27302.md new file mode 100644 index 0000000000..e67a6c8b06 --- /dev/null +++ b/doc/release-notes-27302.md @@ -0,0 +1,4 @@ +Configuration +--- + +- `bitcoind` and `bitcoin-qt` will now raise an error on startup if a datadir that is being used contains a bitcoin.conf file that will be ignored, which can happen when a datadir= line is used in a bitcoin.conf file. The error message is just a diagnostic intended to prevent accidental misconfiguration, and it can be disabled to restore the previous behavior of using the datadir while ignoring the bitcoin.conf contained in it. diff --git a/doc/release-notes-27501.md b/doc/release-notes-27501.md new file mode 100644 index 0000000000..386a00fb34 --- /dev/null +++ b/doc/release-notes-27501.md @@ -0,0 +1,3 @@ +- A new `getprioritisedtransactions` RPC has been added. It returns a map of all fee deltas created by the + user with prioritisetransaction, indexed by txid. The map also indicates whether each transaction is + present in the mempool. diff --git a/doc/release-notes/release-notes-23.2.md b/doc/release-notes/release-notes-23.2.md new file mode 100644 index 0000000000..2d6354ae5d --- /dev/null +++ b/doc/release-notes/release-notes-23.2.md @@ -0,0 +1,72 @@ +23.2 Release Notes +================== + +Bitcoin Core version 23.2 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-23.2/> + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +### P2P + +- #26909 net: prevent peers.dat corruptions by only serializing once +- #27608 p2p: Avoid prematurely clearing download state for other peers +- #27610 Improve performance of p2p inv to send queues + +### Build system + +- #25436 build: suppress array-bounds errors in libxkbcommon +- #25763 bdb: disable Werror for format-security +- #26944 depends: fix systemtap download URL +- #27462 depends: fix compiling bdb with clang-16 on aarch64 + +### Miscellaneous + +- #25444 ci: macOS task imrovements +- #26388 ci: Use macos-ventura-xcode:14.1 image for "macOS native" task +- #26924 refactor: Add missing includes to fix gcc-13 compile error + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Anthony Towns +- Hennadii Stepanov +- MacroFake +- Martin Zumsande +- Michael Ford +- Suhas Daftuar + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/).
\ No newline at end of file diff --git a/doc/release-notes/release-notes-24.1.md b/doc/release-notes/release-notes-24.1.md new file mode 100644 index 0000000000..f46434cc43 --- /dev/null +++ b/doc/release-notes/release-notes-24.1.md @@ -0,0 +1,99 @@ +24.1 Release Notes +================== + +Bitcoin Core version 24.1 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-24.1/> + +This release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +### P2P + +- #26878 I2P network optimizations +- #26909 net: prevent peers.dat corruptions by only serializing once +- #27608 p2p: Avoid prematurely clearing download state for other peers +- #27610 Improve performance of p2p inv to send queues + +### RPC and other APIs + +- #26515 rpc: Require NodeStateStats object in getpeerinfo +- #27279 doc: fix/improve warning helps in {create,load,unload,restore}wallet +- #27468 rest: avoid segfault for invalid URI + +### Build System + +- #26944 depends: fix systemtap download URL +- #27462 depends: fix compiling bdb with clang-16 on aarch64 + +### Wallet + +- #26595 wallet: be able to specify a wallet name and passphrase to migratewallet +- #26675 wallet: For feebump, ignore abandoned descendant spends +- #26679 wallet: Skip rescanning if wallet is more recent than tip +- #26761 wallet: fully migrate address book entries for watchonly/solvable wallets +- #27053 wallet: reuse change dest when re-creating TX with avoidpartialspends +- #27080 wallet: Zero out wallet master key upon locking so it doesn't persist in memory +- #27473 wallet: Properly handle "unknown" Address Type + +### GUI changes + +- gui#687 Load PSBTs using istreambuf_iterator rather than istream_iterator +- gui#704 Correctly limit overview transaction list + +### Miscellaneous + +- #26880 ci: replace Intel macOS CI job +- #26924 refactor: Add missing includes to fix gcc-13 compile error + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- Anthony Towns +- Hennadii Stepanov +- John Moffett +- Jon Atack +- Marco Falke +- Martin Zumsande +- Matthew Zipkin +- Michael Ford +- pablomartin4btc +- Sebastian Falbesoner +- Suhas Daftuar +- Thomas Nguyen +- Vasil Dimov + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/).
\ No newline at end of file diff --git a/doc/release-notes/release-notes-25.0.md b/doc/release-notes/release-notes-25.0.md new file mode 100644 index 0000000000..919cb3b2f3 --- /dev/null +++ b/doc/release-notes/release-notes-25.0.md @@ -0,0 +1,340 @@ +25.0 Release Notes +================== + +Bitcoin Core version 25.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-25.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +- Transactions of non-witness size 65 bytes and above are now allowed by mempool + and relay policy. This is to better reflect the actual afforded protections + against CVE-2017-12842 and open up additional use-cases of smaller transaction sizes. (#26265) + +New RPCs +-------- + +- The scanblocks RPC returns the relevant blockhashes from a set of descriptors by + scanning all blockfilters in the given range. It can be used in combination with + the getblockheader and rescanblockchain RPCs to achieve fast wallet rescans. Note + that this functionality can only be used if a compact block filter index + (-blockfilterindex=1) has been constructed by the node. (#23549) + +Updated RPCs +------------ + +- All JSON-RPC methods accept a new [named + parameter](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md#parameter-passing) called `args` that can + contain positional parameter values. This is a convenience to allow some + parameter values to be passed by name without having to name every value. The + python test framework and `bitcoin-cli` tool both take advantage of this, so + for example: + +```sh +bitcoin-cli -named createwallet wallet_name=mywallet load_on_startup=1 +``` + +Can now be shortened to: + +```sh +bitcoin-cli -named createwallet mywallet load_on_startup=1 +``` + +- The `verifychain` RPC will now return `false` if the checks didn't fail, + but couldn't be completed at the desired depth and level. This could be due + to missing data while pruning, due to an insufficient dbcache or due to + the node being shutdown before the call could finish. (#25574) + +- `sendrawtransaction` has a new, optional argument, `maxburnamount` with a default value of `0`. + Any transaction containing an unspendable output with a value greater than `maxburnamount` will + not be submitted. At present, the outputs deemed unspendable are those with scripts that begin + with an `OP_RETURN` code (known as 'datacarriers'), scripts that exceed the maximum script size, + and scripts that contain invalid opcodes. + +- The `testmempoolaccept` RPC now returns 2 additional results within the "fees" result: + "effective-feerate" is the feerate including fees and sizes of transactions validated together if + package validation was used, and also includes any modified fees from prioritisetransaction. The + "effective-includes" result lists the wtxids of transactions whose modified fees and sizes were used + in the effective-feerate (#26646). + +- `decodescript` may now infer a Miniscript descriptor under P2WSH context if it is not lacking + information. (#27037) + +- `finalizepsbt` is now able to finalize a transaction with inputs spending Miniscript-compatible + P2WSH scripts. (#24149) + +Changes to wallet related RPCs can be found in the Wallet section below. + +Build System +------------ + +- The `--enable-upnp-default` and `--enable-natpmp-default` options + have been removed. If you want to use port mapping, you can + configure it using a .conf file, or by passing the relevant + options at runtime. (#26896) + +Updated settings +---------------- + +- If the `-checkblocks` or `-checklevel` options are explicitly provided by the +user, but the verification checks cannot be completed due to an insufficient +dbcache, Bitcoin Core will now return an error at startup. (#25574) + +- Ports specified in `-port` and `-rpcport` options are now validated at startup. + Values that previously worked and were considered valid can now result in errors. (#22087) + +- Setting `-blocksonly` will now reduce the maximum mempool memory + to 5MB (users may still use `-maxmempool` to override). Previously, + the default 300MB would be used, leading to unexpected memory usage + for users running with `-blocksonly` expecting it to eliminate + mempool memory usage. + + As unused mempool memory is shared with dbcache, this also reduces + the dbcache size for users running with `-blocksonly`, potentially + impacting performance. +- Setting `-maxconnections=0` will now disable `-dnsseed` + and `-listen` (users may still set them to override). + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +New settings +------------ + +- The `shutdownnotify` option is used to specify a command to execute synchronously +before Bitcoin Core has begun its shutdown sequence. (#23395) + + +Wallet +------ + +- The `minconf` option, which allows a user to specify the minimum number +of confirmations a UTXO being spent has, and the `maxconf` option, +which allows specifying the maximum number of confirmations, have been +added to the following RPCs in #25375: + - `fundrawtransaction` + - `send` + - `walletcreatefundedpsbt` + - `sendall` + +- Added a new `next_index` field in the response in `listdescriptors` to + have the same format as `importdescriptors` (#26194) + +- RPC `listunspent` now has a new argument `include_immature_coinbase` + to include coinbase UTXOs that don't meet the minimum spendability + depth requirement (which before were silently skipped). (#25730) + +- Rescans for descriptor wallets are now significantly faster if compact + block filters (BIP158) are available. Since those are not constructed + by default, the configuration option "-blockfilterindex=1" has to be + provided to take advantage of the optimization. This improves the + performance of the RPC calls `rescanblockchain`, `importdescriptors` + and `restorewallet`. (#25957) + +- RPC `unloadwallet` now fails if a rescan is in progress. (#26618) + +- Wallet passphrases may now contain null characters. + Prior to this change, only characters up to the first + null character were recognized and accepted. (#27068) + +- Address Purposes strings are now restricted to the currently known values of "send", + "receive", and "refund". Wallets that have unrecognized purpose strings will have + loading warnings, and the `listlabels` RPC will raise an error if an unrecognized purpose + is requested. (#27217) + +- In the `createwallet`, `loadwallet`, `unloadwallet`, and `restorewallet` RPCs, the + "warning" string field is deprecated in favor of a "warnings" field that + returns a JSON array of strings to better handle multiple warning messages and + for consistency with other wallet RPCs. The "warning" field will be fully + removed from these RPCs in v26. It can be temporarily re-enabled during the + deprecation period by launching bitcoind with the configuration option + `-deprecatedrpc=walletwarningfield`. (#27279) + +- Descriptor wallets can now spend coins sent to P2WSH Miniscript descriptors. (#24149) + +GUI changes +----------- + +- The "Mask values" is a persistent option now. (gui#701) +- The "Mask values" option affects the "Transaction" view now, in addition to the + "Overview" one. (gui#708) + +REST +---- + +- A new `/rest/deploymentinfo` endpoint has been added for fetching various + state info regarding deployments of consensus changes. (#25412) + +Binary verification +---- + +- The binary verification script has been updated. In previous releases it + would verify that the binaries had been signed with a single "release key". + In this release and moving forward it will verify that the binaries are + signed by a _threshold of trusted keys_. For more details and + examples, see: + https://github.com/bitcoin/bitcoin/blob/master/contrib/verify-binaries/README.md + (#27358) + +Low-level changes +================= + +RPC +--- + +- The JSON-RPC server now rejects requests where a parameter is specified multiple + times with the same name, instead of silently overwriting earlier parameter values + with later ones. (#26628) +- RPC `listsinceblock` now accepts an optional `label` argument + to fetch incoming transactions having the specified label. (#25934) +- Previously `setban`, `addpeeraddress`, `walletcreatefundedpsbt`, methods + allowed non-boolean and non-null values to be passed as boolean parameters. + Any string, number, array, or object value that was passed would be treated + as false. After this change, passing any value except `true`, `false`, or + `null` now triggers a JSON value is not of expected type error. (#26213) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- 721217.xyz +- @RandyMcMillan +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Toth +- Anthony Towns +- Antoine Poinsot +- Aurèle Oulès +- Ben Woosley +- Bitcoin Hodler +- brunoerg +- Bushstar +- Carl Dong +- Chris Geihsler +- Cory Fields +- David Gumberg +- dergoegge +- Dhruv Mehta +- Dimitris Tsapakidis +- dougEfish +- Douglas Chimento +- ekzyis +- Elichai Turkel +- Ethan Heilman +- Fabian Jahr +- FractalEncrypt +- furszy +- Gleb Naumenko +- glozow +- Greg Sanders +- Hennadii Stepanov +- hernanmarino +- ishaanam +- ismaelsadeeq +- James O'Beirne +- jdjkelly@gmail.com +- Jeff Ruane +- Jeffrey Czyz +- Jeremy Rubin +- Jesse Barton +- João Barbosa +- JoaoAJMatos +- John Moffett +- Jon Atack +- Jonas Schnelli +- jonatack +- Joshua Kelly +- josibake +- Juan Pablo Civile +- kdmukai +- klementtan +- Kolby ML +- kouloumos +- Kristaps Kaupe +- laanwj +- Larry Ruane +- Leonardo Araujo +- Leonardo Lazzaro +- Luke Dashjr +- MacroFake +- MarcoFalke +- Martin Leitner-Ankerl +- Martin Zumsande +- Matt Whitlock +- Matthew Zipkin +- Michael Ford +- Miles Liu +- mruddy +- Murray Nesbitt +- muxator +- omahs +- pablomartin4btc +- Pasta +- Pieter Wuille +- Pttn +- Randall Naar +- Riahiamirreza +- roconnor-blockstream +- Russell O'Connor +- Ryan Ofsky +- S3RK +- Sebastian Falbesoner +- Seibart Nedor +- sinetek +- Sjors Provoost +- Skuli Dulfari +- SomberNight +- Stacie Waleyko +- stickies-v +- stratospher +- Suhas Daftuar +- Suriyaa Sundararuban +- TheCharlatan +- Vasil Dimov +- Vasil Stoyanov +- virtu +- w0xlt +- willcl-ark +- yancy +- Yusuf Sahin HAMZA + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/).
\ No newline at end of file diff --git a/doc/release-process.md b/doc/release-process.md index 9c2e03d402..930110922c 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -60,6 +60,8 @@ Release Process - Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/17079) for an example) and provide a link to it in the release announcements where useful. - Translations on Transifex - Change the auto-update URL for the new major version's resource away from `master` and to the branch, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/<branch>/src/qt/locale/bitcoin_en.xlf`. Do not forget this or it will keep tracking the translations on master instead, drifting away from the specific major release. +- Prune inputs from the qa-assets repo (See [pruning + inputs](https://github.com/bitcoin-core/qa-assets#pruning-inputs)). #### Before final release @@ -282,7 +284,7 @@ cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.bitcoin-qt/pull/2 - - Push the snap, see https://github.com/bitcoin-core/packaging/blob/master/snap/build.md + - Push the snap, see https://github.com/bitcoin-core/packaging/blob/main/snap/local/build.md - This repo diff --git a/doc/tor.md b/doc/tor.md index 581d124f7a..65aa3ece02 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -2,9 +2,7 @@ It is possible to run Bitcoin Core as a Tor onion service, and connect to such services. -The following directions assume you have a Tor proxy running on port 9050. Many distributions default to having a SOCKS proxy listening on port 9050, but others may not. In particular, the Tor Browser Bundle defaults to listening on port 9150. See [Tor Project FAQ:TBBSocksPort](https://www.torproject.org/docs/faq.html.en#TBBSocksPort) for how to properly -configure Tor. - +The following directions assume you have a Tor proxy running on port 9050. Many distributions default to having a SOCKS proxy listening on port 9050, but others may not. In particular, the Tor Browser Bundle defaults to listening on port 9150. ## Compatibility - Starting with version 22.0, Bitcoin Core only supports Tor version 3 hidden @@ -27,8 +25,7 @@ CLI `-addrinfo` returns the number of addresses known to your node per network. This can be useful to see how many onion peers your node knows, e.g. for `-onlynet=onion`. -To fetch a number of onion addresses that your node knows, for example seven -addresses, use the `getnodeaddresses 7 onion` RPC. +You can use the `getnodeaddresses` RPC to fetch a number of onion peers known to your node; run `bitcoin-cli help getnodeaddresses` for details. ## 1. Run Bitcoin Core behind a Tor proxy diff --git a/doc/tracing.md b/doc/tracing.md index d26cf52fc3..0e3414205a 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -220,7 +220,7 @@ about the transaction. Arguments passed: 1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) -2. Transaction virtual size as `uint64` +2. Transaction virtual size as `int32` 3. Transaction fee as `int64` #### Tracepoint `mempool:removed` @@ -231,7 +231,7 @@ about the transaction. Arguments passed: 1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) 2. Removal reason as `pointer to C-style String` (max. length 9 characters) -3. Transaction virtual size as `uint64` +3. Transaction virtual size as `int32` 4. Transaction fee as `int64` 5. Transaction mempool entry time (epoch) as `uint64` @@ -242,11 +242,11 @@ Passes information about the replaced and replacement transactions. Arguments passed: 1. Replaced transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) -2. Replaced transaction virtual size as `uint64` +2. Replaced transaction virtual size as `int32` 3. Replaced transaction fee as `int64` 4. Replaced transaction mempool entry time (epoch) as `uint64` 5. Replacement transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) -6. Replacement transaction virtual size as `uint64` +6. Replacement transaction virtual size as `int32` 7. Replacement transaction fee as `int64` Note: In cases where a single replacement transaction replaces multiple diff --git a/src/Makefile.am b/src/Makefile.am index d12edca64e..b4ff556eb6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -143,6 +143,8 @@ BITCOIN_CORE_H = \ compat/compat.h \ compat/cpuid.h \ compat/endian.h \ + common/settings.h \ + common/system.h \ compressor.h \ consensus/consensus.h \ consensus/tx_check.h \ @@ -186,6 +188,7 @@ BITCOIN_CORE_H = \ kernel/mempool_limits.h \ kernel/mempool_options.h \ kernel/mempool_persist.h \ + kernel/notifications_interface.h \ kernel/validation_cache_sizes.h \ key.h \ key_io.h \ @@ -214,9 +217,11 @@ BITCOIN_CORE_H = \ node/database_args.h \ node/eviction.h \ node/interface_ui.h \ + node/kernel_notifications.h \ node/mempool_args.h \ node/mempool_persist_args.h \ node/miner.h \ + node/mini_miner.h \ node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ @@ -276,10 +281,13 @@ BITCOIN_CORE_H = \ txorphanage.h \ txrequest.h \ undo.h \ + util/any.h \ util/asmap.h \ + util/batchpriority.h \ util/bip32.h \ util/bitdeque.h \ util/bytevectorhash.h \ + util/chaintype.h \ util/check.h \ util/epochguard.h \ util/error.h \ @@ -292,6 +300,7 @@ BITCOIN_CORE_H = \ util/golombrice.h \ util/hash_type.h \ util/hasher.h \ + util/insert.h \ util/macros.h \ util/message.h \ util/moneystr.h \ @@ -301,13 +310,11 @@ BITCOIN_CORE_H = \ util/readwritefile.h \ util/result.h \ util/serfloat.h \ - util/settings.h \ util/sock.h \ util/spanparsing.h \ util/string.h \ util/syscall_sandbox.h \ util/syserror.h \ - util/system.h \ util/thread.h \ util/threadinterrupt.h \ util/threadnames.h \ @@ -356,7 +363,7 @@ BITCOIN_CORE_H = \ obj/build.h: FORCE @$(MKDIR_P) $(builddir)/obj - @$(top_srcdir)/share/genbuild.sh "$(abs_top_builddir)/src/obj/build.h" \ + $(AM_V_GEN) $(top_srcdir)/share/genbuild.sh "$(abs_top_builddir)/src/obj/build.h" \ "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h @@ -406,9 +413,11 @@ libbitcoin_node_a_SOURCES = \ node/eviction.cpp \ node/interface_ui.cpp \ node/interfaces.cpp \ + node/kernel_notifications.cpp \ node/mempool_args.cpp \ node/mempool_persist_args.cpp \ node/miner.cpp \ + node/mini_miner.cpp \ node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ @@ -654,6 +663,8 @@ libbitcoin_common_a_SOURCES = \ common/init.cpp \ common/interfaces.cpp \ common/run_command.cpp \ + common/settings.cpp \ + common/system.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -705,8 +716,10 @@ libbitcoin_util_a_SOURCES = \ support/cleanse.cpp \ sync.cpp \ util/asmap.cpp \ + util/batchpriority.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ + util/chaintype.cpp \ util/check.cpp \ util/error.cpp \ util/exception.cpp \ @@ -717,12 +730,10 @@ libbitcoin_util_a_SOURCES = \ util/hasher.cpp \ util/sock.cpp \ util/syserror.cpp \ - util/system.cpp \ util/message.cpp \ util/moneystr.cpp \ util/rbf.cpp \ util/readwritefile.cpp \ - util/settings.cpp \ util/thread.cpp \ util/threadinterrupt.cpp \ util/threadnames.cpp \ @@ -901,12 +912,8 @@ libbitcoinkernel_la_SOURCES = \ kernel/bitcoinkernel.cpp \ arith_uint256.cpp \ chain.cpp \ - chainparamsbase.cpp \ - chainparams.cpp \ clientversion.cpp \ coins.cpp \ - common/args.cpp \ - common/config.cpp \ compressor.cpp \ consensus/merkle.cpp \ consensus/tx_check.cpp \ @@ -956,6 +963,8 @@ libbitcoinkernel_la_SOURCES = \ txdb.cpp \ txmempool.cpp \ uint256.cpp \ + util/batchpriority.cpp \ + util/chaintype.cpp \ util/check.cpp \ util/exception.cpp \ util/fs.cpp \ @@ -965,12 +974,10 @@ libbitcoinkernel_la_SOURCES = \ util/moneystr.cpp \ util/rbf.cpp \ util/serfloat.cpp \ - util/settings.cpp \ util/strencodings.cpp \ util/string.cpp \ util/syscall_sandbox.cpp \ util/syserror.cpp \ - util/system.cpp \ util/thread.cpp \ util/threadnames.cpp \ util/time.cpp \ @@ -1046,7 +1053,7 @@ clean-local: -rm -rf test/__pycache__ .rc.o: - @test -f $(WINDRES) + @test -f $(WINDRES) || (echo "windres $(WINDRES) not found, but is required to compile windows resource files"; exit 1) ## FIXME: How to get the appropriate modulename_CPPFLAGS in here? $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ @@ -1101,12 +1108,11 @@ endif %.raw.h: %.raw @$(MKDIR_P) $(@D) - @{ \ + $(AM_V_GEN) { \ echo "static unsigned const char $(*F)_raw[] = {" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};"; \ } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" include Makefile.minisketch.include diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index c230728a1c..c8e510b482 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -47,6 +47,7 @@ bench_bench_bitcoin_SOURCES = \ bench/rollingbloom.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ + bench/streams_findbyte.cpp \ bench/strencodings.cpp \ bench/util_time.cpp \ bench/verify_script.cpp diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 602a118259..7852d1a2fa 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -371,13 +371,13 @@ translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCO @rm -f $(srcdir)/qt/locale/bitcoin_en.xlf.old $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) - @test -f $(RCC) + @test -f $(RCC) || (echo "rcc $(RCC) not found, but is required for generating qrc cpp files"; exit 1) @cp -f $< $(@D)/temp_$(<F) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin_locale --format-version 1 $(@D)/temp_$(<F) > $@ @rm $(@D)/temp_$(<F) $(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(QT_RES_FONTS) $(QT_RES_ICONS) $(QT_RES_ANIMATION) - @test -f $(RCC) + @test -f $(RCC) || (echo "rcc $(RCC) not found, but is required for generating qrc cpp files"; exit 1) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin --format-version 1 $< > $@ CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno qt/temp_bitcoin_locale.qrc @@ -404,7 +404,7 @@ bitcoin_qt_apk: FORCE cd qt/android && ./gradlew build ui_%.h: %.ui - @test -f $(UIC) + @test -f $(UIC) || (echo "uic $(UIC) not found, but is required for generating ui headers"; exit 1) @$(MKDIR_P) $(@D) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(UIC) -o $@ $< || (echo "Error creating $@"; false) @@ -415,6 +415,6 @@ moc_%.cpp: %.h $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(MOC) $(DEFAULT_INCLUDES) $(QT_INCLUDES_UNSUPPRESSED) $(MOC_DEFS) $< > $@ %.qm: %.ts - @test -f $(LRELEASE) + @test -f $(LRELEASE) || (echo "lrelease $(LRELEASE) not found, but is required for generating translations"; exit 1) @$(MKDIR_P) $(@D) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LRELEASE) -silent $< -qm $@ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 69965ed1b8..8e45c797bf 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -106,6 +106,7 @@ BITCOIN_TESTS =\ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ test/miner_tests.cpp \ + test/miniminer_tests.cpp \ test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ @@ -287,6 +288,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/message.cpp \ test/fuzz/miniscript.cpp \ test/fuzz/minisketch.cpp \ + test/fuzz/mini_miner.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ test/fuzz/net.cpp \ @@ -344,6 +346,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/txorphan.cpp \ test/fuzz/txrequest.cpp \ test/fuzz/utxo_snapshot.cpp \ + test/fuzz/utxo_total_supply.cpp \ test/fuzz/validation_load_mempool.cpp \ test/fuzz/versionbits.cpp endif # ENABLE_FUZZ_BINARY @@ -421,10 +424,9 @@ endif %.json.h: %.json @$(MKDIR_P) $(@D) - @{ \ + $(AM_V_GEN) { \ echo "namespace json_tests{" && \ echo "static unsigned const char $(*F)[] = {" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" diff --git a/src/addrdb.cpp b/src/addrdb.cpp index b679ad0602..cb1c49050e 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -9,6 +9,7 @@ #include <chainparams.h> #include <clientversion.h> #include <common/args.h> +#include <common/settings.h> #include <cstdint> #include <hash.h> #include <logging.h> @@ -21,7 +22,6 @@ #include <univalue.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/settings.h> #include <util/translation.h> namespace { @@ -132,7 +132,7 @@ CBanDB::CBanDB(fs::path ban_list_path) bool CBanDB::Write(const banmap_t& banSet) { std::vector<std::string> errors; - if (util::WriteSettings(m_banlist_json, {{JSON_KEY, BanMapToJson(banSet)}}, errors)) { + if (common::WriteSettings(m_banlist_json, {{JSON_KEY, BanMapToJson(banSet)}}, errors)) { return true; } @@ -152,10 +152,10 @@ bool CBanDB::Read(banmap_t& banSet) return false; } - std::map<std::string, util::SettingsValue> settings; + std::map<std::string, common::SettingsValue> settings; std::vector<std::string> errors; - if (!util::ReadSettings(m_banlist_json, settings, errors)) { + if (!common::ReadSettings(m_banlist_json, settings, errors)) { for (const auto& err : errors) { LogPrintf("Cannot load banlist %s: %s\n", fs::PathToString(m_banlist_json), err); } @@ -183,10 +183,10 @@ void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) DeserializeDB(ssPeers, addr, false); } -std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) +util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args) { auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + auto addrman{std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman)}; const auto start{SteadyClock::now()}; const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -200,19 +200,18 @@ std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, con DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { if (!RenameOver(path_addr, (fs::path)path_addr + ".bak")) { - addrman = nullptr; - return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); + return util::Error{strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again."))}; } // Addrman can be in an inconsistent state after failure, reset it addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { - addrman = nullptr; - return strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), - e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr))); + return util::Error{strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), + e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr)))}; } - return std::nullopt; + return {std::move(addrman)}; // std::move should be unneccessary but is temporarily needed to work around clang bug + // (https://github.com/bitcoin/bitcoin/pull/25977#issuecomment-1561270092) } void DumpAnchors(const fs::path& anchors_db_path, const std::vector<CAddress>& anchors) diff --git a/src/addrdb.h b/src/addrdb.h index 08d86d0f01..0037495d18 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -6,11 +6,11 @@ #ifndef BITCOIN_ADDRDB_H #define BITCOIN_ADDRDB_H -#include <net_types.h> // For banmap_t -#include <univalue.h> +#include <net_types.h> #include <util/fs.h> +#include <util/result.h> -#include <optional> +#include <memory> #include <vector> class ArgsManager; @@ -18,7 +18,6 @@ class AddrMan; class CAddress; class CDataStream; class NetGroupManager; -struct bilingual_str; bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); /** Only used by tests. */ @@ -49,7 +48,7 @@ public: }; /** Returns an error string on failure */ -std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman); +util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args); /** * Dump the anchor IP address database (anchors.dat) diff --git a/src/attributes.h b/src/attributes.h index 9957bcd84b..a4603b0270 100644 --- a/src/attributes.h +++ b/src/attributes.h @@ -16,4 +16,12 @@ # define LIFETIMEBOUND #endif +#if defined(__GNUC__) +# define ALWAYS_INLINE inline __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define ALWAYS_INLINE __forceinline +#else +# error No known always_inline attribute for this platform. +#endif + #endif // BITCOIN_ATTRIBUTES_H diff --git a/src/banman.cpp b/src/banman.cpp index 5b2049d654..a96b7e3c53 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -5,11 +5,11 @@ #include <banman.h> +#include <common/system.h> #include <logging.h> #include <netaddress.h> #include <node/interface_ui.h> #include <sync.h> -#include <util/system.h> #include <util/time.h> #include <util/translation.h> diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 8a5cab443f..f044feebba 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -72,20 +72,6 @@ static void FillAddrMan(AddrMan& addrman) AddAddressesToAddrMan(addrman); } -static CNetAddr ResolveIP(const std::string& ip) -{ - CNetAddr addr; - LookupHost(ip, addr, false); - return addr; -} - -static CService ResolveService(const std::string& ip, uint16_t port = 0) -{ - CService serv; - Lookup(ip, serv, port, false); - return serv; -} - /* Benchmarks */ static void AddrManAdd(benchmark::Bench& bench) @@ -118,8 +104,8 @@ static void AddrManSelectFromAlmostEmpty(benchmark::Bench& bench) AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; // Add one address to the new table - CService addr = ResolveService("250.3.1.1", 8333); - addrman.Add({CAddress(addr, NODE_NONE)}, ResolveService("250.3.1.1", 8333)); + CService addr = Lookup("250.3.1.1", 8333, false).value(); + addrman.Add({CAddress(addr, NODE_NONE)}, addr); bench.run([&] { (void)addrman.Select(); @@ -135,7 +121,7 @@ static void AddrManSelectByNetwork(benchmark::Bench& bench) i2p_service.SetSpecial("udhdrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v4jna.b32.i2p"); CAddress i2p_address(i2p_service, NODE_NONE); i2p_address.nTime = Now<NodeSeconds>(); - CNetAddr source = ResolveIP("252.2.2.2"); + const CNetAddr source{LookupHost("252.2.2.2", false).value()}; addrman.Add({i2p_address}, source); FillAddrMan(addrman); diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 8dd4117a3e..4d032cefc5 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -27,7 +27,7 @@ static void AssembleBlock(benchmark::Bench& bench) std::array<CTransactionRef, NUM_BLOCKS - COINBASE_MATURITY + 1> txs; for (size_t b{0}; b < NUM_BLOCKS; ++b) { CMutableTransaction tx; - tx.vin.push_back(MineBlock(test_setup->m_node, P2WSH_OP_TRUE)); + tx.vin.push_back(CTxIn{MineBlock(test_setup->m_node, P2WSH_OP_TRUE)}); tx.vin.back().scriptWitness = witness; tx.vout.emplace_back(1337, P2WSH_OP_TRUE); if (NUM_BLOCKS - b >= COINBASE_MATURITY) diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 260c8991ce..269ac847a5 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -9,6 +9,7 @@ #include <common/args.h> #include <consensus/validation.h> #include <streams.h> +#include <util/chaintype.h> #include <validation.h> // These are the two major time-sinks which happen after we have fully received @@ -36,7 +37,7 @@ static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) stream.write({&a, 1}); // Prevent compaction ArgsManager bench_args; - const auto chainParams = CreateChainParams(bench_args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(bench_args, ChainType::MAIN); bench.unit("block").run([&] { CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 8ad6fde6bf..70e0b86eba 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -4,11 +4,11 @@ #include <bench/bench.h> #include <checkqueue.h> +#include <common/system.h> #include <key.h> #include <prevector.h> #include <pubkey.h> #include <random.h> -#include <util/system.h> #include <vector> diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 11ce14728d..0e110a653a 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -9,6 +9,7 @@ #include <wallet/coinselection.h> #include <wallet/spend.h> #include <wallet/wallet.h> +#include <wallet/test/util.h> #include <set> @@ -20,7 +21,7 @@ using wallet::CWallet; using wallet::CWalletTx; using wallet::CoinEligibilityFilter; using wallet::CoinSelectionParams; -using wallet::CreateDummyWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::OutputGroup; using wallet::SelectCoinsBnB; using wallet::TxStateInactive; @@ -46,7 +47,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateMockableWalletDatabase()); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp index 0fd842c7c3..2ff72a3012 100644 --- a/src/bench/load_external.cpp +++ b/src/bench/load_external.cpp @@ -6,6 +6,7 @@ #include <bench/data.h> #include <chainparams.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <validation.h> /** @@ -22,7 +23,7 @@ */ static void LoadExternalBlockFile(benchmark::Bench& bench) { - const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN)}; + const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)}; // Create a single block as in the blocks files (magic bytes, block size, // block data) as a stream object. diff --git a/src/bench/logging.cpp b/src/bench/logging.cpp index 9aedb26236..c97c4e151b 100644 --- a/src/bench/logging.cpp +++ b/src/bench/logging.cpp @@ -5,6 +5,7 @@ #include <bench/bench.h> #include <logging.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> // All but 2 of the benchmarks should have roughly similar performance: // @@ -18,7 +19,7 @@ static void Logging(benchmark::Bench& bench, const std::vector<const char*>& ext LogInstance().DisableCategory(BCLog::LogFlags::ALL); TestingSetup test_setup{ - CBaseChainParams::REGTEST, + ChainType::REGTEST, extra_args, }; diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 80c959cdfb..826da73800 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -7,6 +7,7 @@ #include <policy/policy.h> #include <test/util/setup_common.h> #include <txmempool.h> +#include <util/chaintype.h> #include <validation.h> #include <vector> @@ -88,7 +89,7 @@ static void ComplexMemPool(benchmark::Bench& bench) childTxs = static_cast<int>(bench.complexityN()); } std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /*min_ancestors=*/1); - const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN); + const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN); CTxMemPool& pool = *testing_setup.get()->m_node.mempool; LOCK2(cs_main, pool.cs); bench.run([&]() NO_THREAD_SAFETY_ANALYSIS { @@ -103,7 +104,7 @@ static void ComplexMemPool(benchmark::Bench& bench) static void MempoolCheck(benchmark::Bench& bench) { FastRandomContext det_rand{true}; - auto testing_setup = MakeNoLogFileContext<TestChain100Setup>(CBaseChainParams::REGTEST, {"-checkmempool=1"}); + auto testing_setup = MakeNoLogFileContext<TestChain100Setup>(ChainType::REGTEST, {"-checkmempool=1"}); CTxMemPool& pool = *testing_setup.get()->m_node.mempool; LOCK2(cs_main, pool.cs); testing_setup->PopulateMempool(det_rand, 400, true); diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index f68b6acb5b..a9b197b190 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -8,6 +8,7 @@ #include <rpc/blockchain.h> #include <streams.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <validation.h> #include <univalue.h> @@ -15,7 +16,7 @@ namespace { struct TestBlockAndIndex { - const std::unique_ptr<const TestingSetup> testing_setup{MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN)}; + const std::unique_ptr<const TestingSetup> testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)}; CBlock block{}; uint256 blockHash{}; CBlockIndex blockindex{}; diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index e3e1a07c83..7e274370e0 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -3,12 +3,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <chainparamsbase.h> #include <kernel/cs_main.h> #include <kernel/mempool_entry.h> #include <rpc/mempool.h> #include <test/util/setup_common.h> #include <txmempool.h> +#include <util/chaintype.h> #include <univalue.h> @@ -21,7 +21,7 @@ static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& poo static void RpcMempool(benchmark::Bench& bench) { - const auto testing_setup = MakeNoLogFileContext<const ChainTestingSetup>(CBaseChainParams::MAIN); + const auto testing_setup = MakeNoLogFileContext<const ChainTestingSetup>(ChainType::MAIN); CTxMemPool& pool = *Assert(testing_setup->m_node.mempool); LOCK2(cs_main, pool.cs); diff --git a/src/bench/streams_findbyte.cpp b/src/bench/streams_findbyte.cpp new file mode 100644 index 0000000000..77f5940926 --- /dev/null +++ b/src/bench/streams_findbyte.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> + +#include <util/fs.h> +#include <streams.h> + +static void FindByte(benchmark::Bench& bench) +{ + // Setup + FILE* file = fsbridge::fopen("streams_tmp", "w+b"); + const size_t file_size = 200; + uint8_t data[file_size] = {0}; + data[file_size-1] = 1; + fwrite(&data, sizeof(uint8_t), file_size, file); + rewind(file); + CBufferedFile bf(file, /*nBufSize=*/file_size + 1, /*nRewindIn=*/file_size, 0, 0); + + bench.run([&] { + bf.SetPos(0); + bf.FindByte(std::byte(1)); + }); + + // Cleanup + bf.fclose(); + fs::remove("streams_tmp"); +} + +BENCHMARK(FindByte, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/util_time.cpp b/src/bench/util_time.cpp index 8dbbdec28c..4cbc0dfbbd 100644 --- a/src/bench/util_time.cpp +++ b/src/bench/util_time.cpp @@ -32,7 +32,7 @@ static void BenchTimeMillis(benchmark::Bench& bench) static void BenchTimeMillisSys(benchmark::Bench& bench) { bench.run([&] { - (void)GetTimeMillis(); + (void)TicksSinceEpoch<std::chrono::milliseconds>(SystemClock::now()); }); } diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index d5d057e96d..bf2195293e 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <interfaces/chain.h> +#include <node/chainstate.h> #include <node/context.h> #include <test/util/mining.h> #include <test/util/setup_common.h> @@ -14,26 +15,21 @@ #include <optional> -using wallet::CWallet; -using wallet::CreateMockWalletDatabase; -using wallet::DBErrors; -using wallet::GetBalance; -using wallet::WALLET_FLAG_DESCRIPTORS; - -const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; - +namespace wallet { static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_mine) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + // Set clock to genesis block, so the descriptors/keys creation time don't interfere with the blocks scanning process. + // The reason is 'generatetoaddress', which creates a chain with deterministic timestamps in the past. + SetMockTime(test_setup->m_node.chainman->GetParams().GenesisBlock().nTime); + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetupDescriptorScriptPubKeyMans(); - if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); } auto handler = test_setup->m_node.chain->handleNotifications({&wallet, [](CWallet*) {}}); @@ -63,3 +59,4 @@ BENCHMARK(WalletBalanceDirty, benchmark::PriorityLevel::HIGH); BENCHMARK(WalletBalanceClean, benchmark::PriorityLevel::HIGH); BENCHMARK(WalletBalanceMine, benchmark::PriorityLevel::HIGH); BENCHMARK(WalletBalanceWatch, benchmark::PriorityLevel::HIGH); +} // namespace wallet diff --git a/src/bench/wallet_create_tx.cpp b/src/bench/wallet_create_tx.cpp index cb31421598..13b0019fd2 100644 --- a/src/bench/wallet_create_tx.cpp +++ b/src/bench/wallet_create_tx.cpp @@ -15,7 +15,7 @@ #include <wallet/wallet.h> using wallet::CWallet; -using wallet::CreateMockWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::DBErrors; using wallet::WALLET_FLAG_DESCRIPTORS; @@ -83,7 +83,7 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -136,7 +136,7 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type static void AvailableCoins(benchmark::Bench& bench, const std::vector<OutputType>& output_type) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index 6b09adcc9d..5453238728 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -16,33 +16,7 @@ #include <optional> -using wallet::CWallet; -using wallet::DatabaseFormat; -using wallet::DatabaseOptions; -using wallet::TxStateInactive; -using wallet::WALLET_FLAG_DESCRIPTORS; -using wallet::WalletContext; -using wallet::WalletDatabase; - -static std::shared_ptr<CWallet> BenchLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, DatabaseOptions& options) -{ - bilingual_str error; - std::vector<bilingual_str> warnings; - auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); - NotifyWalletLoaded(context, wallet); - if (context.chain) { - wallet->postInitProcess(); - } - return wallet; -} - -static void BenchUnloadWallet(std::shared_ptr<CWallet>&& wallet) -{ - SyncWithValidationInterfaceQueue(); - wallet->m_chain_notifications_handler.reset(); - UnloadWallet(std::move(wallet)); -} - +namespace wallet{ static void AddTx(CWallet& wallet) { CMutableTransaction mtx; @@ -55,7 +29,6 @@ static void AddTx(CWallet& wallet) static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) { const auto test_setup = MakeNoLogFileContext<TestingSetup>(); - test_setup->m_args.ForceSetArg("-unsafesqlitesync", "1"); WalletContext context; context.args = &test_setup->m_args; @@ -63,32 +36,29 @@ static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) // Setup the wallet // Loading the wallet will also create it - DatabaseOptions options; - if (legacy_wallet) { - options.require_format = DatabaseFormat::BERKELEY; - } else { - options.create_flags = WALLET_FLAG_DESCRIPTORS; - options.require_format = DatabaseFormat::SQLITE; + uint64_t create_flags = 0; + if (!legacy_wallet) { + create_flags = WALLET_FLAG_DESCRIPTORS; } - auto database = CreateMockWalletDatabase(options); - auto wallet = BenchLoadWallet(std::move(database), context, options); + auto database = CreateMockableWalletDatabase(); + auto wallet = TestLoadWallet(std::move(database), context, create_flags); // Generate a bunch of transactions and addresses to put into the wallet for (int i = 0; i < 1000; ++i) { AddTx(*wallet); } - database = DuplicateMockDatabase(wallet->GetDatabase(), options); + database = DuplicateMockDatabase(wallet->GetDatabase()); // reload the wallet for the actual benchmark - BenchUnloadWallet(std::move(wallet)); + TestUnloadWallet(std::move(wallet)); bench.epochs(5).run([&] { - wallet = BenchLoadWallet(std::move(database), context, options); + wallet = TestLoadWallet(std::move(database), context, create_flags); // Cleanup - database = DuplicateMockDatabase(wallet->GetDatabase(), options); - BenchUnloadWallet(std::move(wallet)); + database = DuplicateMockDatabase(wallet->GetDatabase()); + TestUnloadWallet(std::move(wallet)); }); } @@ -101,3 +71,4 @@ BENCHMARK(WalletLoadingLegacy, benchmark::PriorityLevel::HIGH); static void WalletLoadingDescriptors(benchmark::Bench& bench) { WalletLoading(bench, /*legacy_wallet=*/false); } BENCHMARK(WalletLoadingDescriptors, benchmark::PriorityLevel::HIGH); #endif +} // namespace wallet diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 04f6aae4f7..432bdc8e33 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -12,12 +12,11 @@ // It is part of the libbitcoinkernel project. #include <kernel/chainparams.h> +#include <kernel/chainstatemanager_opts.h> #include <kernel/checks.h> #include <kernel/context.h> #include <kernel/validation_cache_sizes.h> -#include <chainparams.h> -#include <common/args.h> #include <consensus/validation.h> #include <core_io.h> #include <node/blockstorage.h> @@ -25,14 +24,18 @@ #include <node/chainstate.h> #include <scheduler.h> #include <script/sigcache.h> +#include <util/chaintype.h> #include <util/thread.h> #include <validation.h> #include <validationinterface.h> #include <cassert> +#include <cstdint> #include <filesystem> #include <functional> #include <iosfwd> +#include <memory> +#include <string> int main(int argc, char* argv[]) { @@ -48,18 +51,14 @@ int main(int argc, char* argv[]) } std::filesystem::path abs_datadir = std::filesystem::absolute(argv[1]); std::filesystem::create_directories(abs_datadir); - gArgs.ForceSetArg("-datadir", abs_datadir.string()); - // SETUP: Misc Globals - SelectParams(CBaseChainParams::MAIN); - auto chainparams = CChainParams::Main(); - + // SETUP: Context kernel::Context kernel_context{}; // We can't use a goto here, but we can use an assert since none of the // things instantiated so far requires running the epilogue to be torn down // properly - assert(!kernel::SanityChecks(kernel_context).has_value()); + assert(kernel::SanityChecks(kernel_context)); // Necessary for CheckInputScripts (eventually called by ProcessNewBlock), // which will try the script cache first and fall back to actually @@ -79,14 +78,42 @@ int main(int argc, char* argv[]) GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + class KernelNotifications : public kernel::Notifications + { + public: + void blockTip(SynchronizationState, CBlockIndex&) override + { + std::cout << "Block tip changed" << std::endl; + } + void headerTip(SynchronizationState, int64_t height, int64_t timestamp, bool presync) override + { + std::cout << "Header tip changed: " << height << ", " << timestamp << ", " << presync << std::endl; + } + void progress(const bilingual_str& title, int progress_percent, bool resume_possible) override + { + std::cout << "Progress: " << title.original << ", " << progress_percent << ", " << resume_possible << std::endl; + } + void warning(const bilingual_str& warning) override + { + std::cout << "Warning: " << warning.original << std::endl; + } + }; + auto notifications = std::make_unique<KernelNotifications>(); + // SETUP: Chainstate + auto chainparams = CChainParams::Main(); const ChainstateManager::Options chainman_opts{ .chainparams = *chainparams, - .datadir = gArgs.GetDataDirNet(), + .datadir = abs_datadir, .adjusted_time_callback = NodeClock::now, + .notifications = *notifications, + }; + const node::BlockManager::Options blockman_opts{ + .chainparams = chainman_opts.chainparams, + .blocks_dir = abs_datadir / "blocks", }; - ChainstateManager chainman{chainman_opts, {}}; + ChainstateManager chainman{chainman_opts, blockman_opts}; node::CacheSizes cache_sizes; cache_sizes.block_tree_db = 2 << 20; @@ -117,7 +144,8 @@ int main(int argc, char* argv[]) // Main program logic starts here std::cout << "Hello! I'm going to print out some information about your datadir." << std::endl - << "\t" << "Path: " << gArgs.GetDataDirNet() << std::endl; + << "\t" + << "Path: " << abs_datadir << std::endl; { LOCK(chainman.GetMutex()); std::cout diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 63ba6a935d..45db7a9a66 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -10,6 +10,7 @@ #include <chainparamsbase.h> #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <common/url.h> #include <compat/compat.h> #include <compat/stdin.h> @@ -20,9 +21,9 @@ #include <rpc/request.h> #include <tinyformat.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/exception.h> #include <util/strencodings.h> -#include <util/system.h> #include <util/time.h> #include <util/translation.h> @@ -73,10 +74,10 @@ static void SetupCliArgs(ArgsManager& argsman) { SetupHelpOptions(argsman); - const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); - const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); - const auto signetBaseParams = CreateBaseChainParams(CBaseChainParams::SIGNET); - const auto regtestBaseParams = CreateBaseChainParams(CBaseChainParams::REGTEST); + const auto defaultBaseParams = CreateBaseChainParams(ChainType::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(ChainType::TESTNET); + const auto signetBaseParams = CreateBaseChainParams(ChainType::SIGNET); + const auto regtestBaseParams = CreateBaseChainParams(ChainType::REGTEST); argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -174,7 +175,7 @@ static int AppInitRPC(int argc, char* argv[]) } // Check for chain settings (BaseParams() calls are only valid after this clause) try { - SelectBaseParams(gArgs.GetChainName()); + SelectBaseParams(gArgs.GetChainType()); } catch (const std::exception& e) { tfm::format(std::cerr, "Error: %s\n", e.what()); return EXIT_FAILURE; @@ -426,10 +427,17 @@ private: std::vector<Peer> m_peers; std::string ChainToString() const { - if (gArgs.GetChainName() == CBaseChainParams::TESTNET) return " testnet"; - if (gArgs.GetChainName() == CBaseChainParams::SIGNET) return " signet"; - if (gArgs.GetChainName() == CBaseChainParams::REGTEST) return " regtest"; - return ""; + switch (gArgs.GetChainType()) { + case ChainType::TESTNET: + return " testnet"; + case ChainType::SIGNET: + return " signet"; + case ChainType::REGTEST: + return " regtest"; + case ChainType::MAIN: + return ""; + } + assert(false); } std::string PingTimeToString(double seconds) const { @@ -863,7 +871,7 @@ static UniValue ConnectAndCallRPC(BaseRequestHandler* rh, const std::string& str try { response = CallRPC(rh, strMethod, args, rpcwallet); if (fWait) { - const UniValue& error = find_value(response, "error"); + const UniValue& error = response.find_value("error"); if (!error.isNull() && error["code"].getInt<int>() == RPC_IN_WARMUP) { throw CConnectionFailed("server in warmup"); } @@ -891,8 +899,8 @@ static void ParseResult(const UniValue& result, std::string& strPrint) static void ParseError(const UniValue& error, std::string& strPrint, int& nRet) { if (error.isObject()) { - const UniValue& err_code = find_value(error, "code"); - const UniValue& err_msg = find_value(error, "message"); + const UniValue& err_code = error.find_value("code"); + const UniValue& err_msg = error.find_value("message"); if (!err_code.isNull()) { strPrint = "error code: " + err_code.getValStr() + "\n"; } @@ -918,15 +926,15 @@ static void GetWalletBalances(UniValue& result) { DefaultRequestHandler rh; const UniValue listwallets = ConnectAndCallRPC(&rh, "listwallets", /* args=*/{}); - if (!find_value(listwallets, "error").isNull()) return; - const UniValue& wallets = find_value(listwallets, "result"); + if (!listwallets.find_value("error").isNull()) return; + const UniValue& wallets = listwallets.find_value("result"); if (wallets.size() <= 1) return; UniValue balances(UniValue::VOBJ); for (const UniValue& wallet : wallets.getValues()) { const std::string& wallet_name = wallet.get_str(); const UniValue getbalances = ConnectAndCallRPC(&rh, "getbalances", /* args=*/{}, wallet_name); - const UniValue& balance = find_value(getbalances, "result")["mine"]["trusted"]; + const UniValue& balance = getbalances.find_value("result")["mine"]["trusted"]; balances.pushKV(wallet_name, balance); } result.pushKV("balances", balances); @@ -962,7 +970,7 @@ static void GetProgressBar(double progress, std::string& progress_bar) */ static void ParseGetInfoResult(UniValue& result) { - if (!find_value(result, "error").isNull()) return; + if (!result.find_value("error").isNull()) return; std::string RESET, GREEN, BLUE, YELLOW, MAGENTA, CYAN; bool should_colorize = false; @@ -1174,9 +1182,9 @@ static int CommandLineRPC(int argc, char *argv[]) rh.reset(new NetinfoRequestHandler()); } else if (gArgs.GetBoolArg("-generate", false)) { const UniValue getnewaddress{GetNewAddress()}; - const UniValue& error{find_value(getnewaddress, "error")}; + const UniValue& error{getnewaddress.find_value("error")}; if (error.isNull()) { - SetGenerateToAddressArgs(find_value(getnewaddress, "result").get_str(), args); + SetGenerateToAddressArgs(getnewaddress.find_value("result").get_str(), args); rh.reset(new GenerateToAddressRequestHandler()); } else { ParseError(error, strPrint, nRet); @@ -1198,8 +1206,8 @@ static int CommandLineRPC(int argc, char *argv[]) const UniValue reply = ConnectAndCallRPC(rh.get(), method, args, wallet_name); // Parse reply - UniValue result = find_value(reply, "result"); - const UniValue& error = find_value(reply, "error"); + UniValue result = reply.find_value("result"); + const UniValue& error = reply.find_value("error"); if (error.isNull()) { if (gArgs.GetBoolArg("-getinfo", false)) { if (!gArgs.IsArgSet("-rpcwallet")) { diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index c35a111313..0c25ddf373 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -6,9 +6,11 @@ #include <config/bitcoin-config.h> #endif +#include <chainparamsbase.h> #include <clientversion.h> #include <coins.h> #include <common/args.h> +#include <common/system.h> #include <compat/compat.h> #include <consensus/amount.h> #include <consensus/consensus.h> @@ -26,7 +28,6 @@ #include <util/rbf.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <cstdio> @@ -91,7 +92,7 @@ static int AppInitRawTx(int argc, char* argv[]) // Check for chain settings (Params() calls are only valid after this clause) try { - SelectParams(gArgs.GetChainName()); + SelectParams(gArgs.GetChainType()); } catch (const std::exception& e) { tfm::format(std::cerr, "Error: %s\n", e.what()); return EXIT_FAILURE; diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index e3f70e1b76..582c18c9c6 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -12,11 +12,11 @@ #include <chainparamsbase.h> #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <compat/compat.h> #include <core_io.h> #include <streams.h> #include <util/exception.h> -#include <util/system.h> #include <util/translation.h> #include <version.h> @@ -75,7 +75,7 @@ static int AppInitUtil(ArgsManager& args, int argc, char* argv[]) // Check for chain settings (Params() calls are only valid after this clause) try { - SelectParams(args.GetChainName()); + SelectParams(args.GetChainType()); } catch (const std::exception& e) { tfm::format(std::cerr, "Error: %s\n", e.what()); return EXIT_FAILURE; diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 5dbf8a8616..d5dfbbec27 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -10,6 +10,7 @@ #include <chainparamsbase.h> #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <common/url.h> #include <compat/compat.h> #include <interfaces/init.h> @@ -18,7 +19,6 @@ #include <pubkey.h> #include <tinyformat.h> #include <util/exception.h> -#include <util/system.h> #include <util/translation.h> #include <wallet/wallettool.h> @@ -91,7 +91,7 @@ static std::optional<int> WalletAppInit(ArgsManager& args, int argc, char* argv[ return EXIT_FAILURE; } // Check for chain settings (Params() calls are only valid after this clause) - SelectParams(args.GetChainName()); + SelectParams(args.GetChainType()); return std::nullopt; } diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index e476b06017..c561f9aa14 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -11,6 +11,7 @@ #include <clientversion.h> #include <common/args.h> #include <common/init.h> +#include <common/system.h> #include <common/url.h> #include <compat/compat.h> #include <init.h> @@ -25,7 +26,6 @@ #include <util/strencodings.h> #include <util/syscall_sandbox.h> #include <util/syserror.h> -#include <util/system.h> #include <util/threadnames.h> #include <util/tokenpipe.h> #include <util/translation.h> @@ -112,20 +112,30 @@ int fork_daemon(bool nochdir, bool noclose, TokenPipeEnd& endpoint) #endif -static bool AppInit(NodeContext& node, int argc, char* argv[]) +static bool ParseArgs(ArgsManager& args, int argc, char* argv[]) { - bool fRet = false; - - util::ThreadSetInternalName("init"); - // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() - ArgsManager& args = *Assert(node.args); SetupServerArgs(args); std::string error; if (!args.ParseParameters(argc, argv, error)) { return InitError(Untranslated(strprintf("Error parsing command line arguments: %s", error))); } + if (auto error = common::InitConfig(args)) { + return InitError(error->message, error->details); + } + + // Error out when loose non-argument tokens are encountered on command line + for (int i = 1; i < argc; i++) { + if (!IsSwitchChar(argv[i][0])) { + return InitError(Untranslated(strprintf("Command line contains unexpected token '%s', see bitcoind -h for a list of options.", argv[i]))); + } + } + return true; +} + +static bool ProcessInitCommands(ArgsManager& args) +{ // Process help and version before taking care about datadir if (HelpRequested(args) || args.IsArgSet("-version")) { std::string strUsage = PACKAGE_NAME " version " + FormatFullVersion() + "\n"; @@ -142,6 +152,14 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) return true; } + return false; +} + +static bool AppInit(NodeContext& node) +{ + bool fRet = false; + ArgsManager& args = *Assert(node.args); + #if HAVE_DECL_FORK // Communication with parent after daemonizing. This is used for signalling in the following ways: // - a boolean token is sent when the initialization process (all the Init* functions) have finished to indicate @@ -153,23 +171,12 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) std::any context{&node}; try { - if (auto error = common::InitConfig(args)) { - return InitError(error->message, error->details); - } - - // Error out when loose non-argument tokens are encountered on command line - for (int i = 1; i < argc; i++) { - if (!IsSwitchChar(argv[i][0])) { - return InitError(Untranslated(strprintf("Command line contains unexpected token '%s', see bitcoind -h for a list of options.", argv[i]))); - } - } - // -server defaults to true for bitcoind but not for the GUI so do this here args.SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console InitLogging(args); InitParameterInteraction(args); - if (!AppInitBasicSetup(args)) { + if (!AppInitBasicSetup(args, node.exit_status)) { // InitError will have been called with detailed error, which ends up on console return false; } @@ -236,12 +243,6 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) } #endif SetSyscallSandboxPolicy(SyscallSandboxPolicy::SHUTOFF); - if (fRet) { - WaitForShutdown(); - } - Interrupt(node); - Shutdown(node); - return fRet; } @@ -264,5 +265,22 @@ MAIN_FUNCTION // Connect bitcoind signal handlers noui_connect(); - return (AppInit(node, argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE); + util::ThreadSetInternalName("init"); + + // Interpret command line arguments + ArgsManager& args = *Assert(node.args); + if (!ParseArgs(args, argc, argv)) return EXIT_FAILURE; + // Process early info return commands such as -help or -version + if (ProcessInitCommands(args)) return EXIT_SUCCESS; + + // Start application + if (AppInit(node)) { + WaitForShutdown(); + } else { + node.exit_status = EXIT_FAILURE; + } + Interrupt(node); + Shutdown(node); + + return node.exit_status; } diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index a29e4f794e..9aa0a6ba20 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -3,16 +3,16 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <blockencodings.h> +#include <chainparams.h> +#include <common/system.h> #include <consensus/consensus.h> #include <consensus/validation.h> -#include <chainparams.h> #include <crypto/sha256.h> #include <crypto/siphash.h> #include <random.h> #include <streams.h> #include <txmempool.h> #include <validation.h> -#include <util/system.h> #include <unordered_map> diff --git a/src/chainparams.cpp b/src/chainparams.cpp index d3d358a3f0..539578085b 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -5,16 +5,21 @@ #include <chainparams.h> -#include <chainparamsseeds.h> +#include <chainparamsbase.h> #include <common/args.h> -#include <consensus/merkle.h> +#include <consensus/params.h> #include <deploymentinfo.h> -#include <hash.h> // for signet block challenge hash #include <logging.h> -#include <script/interpreter.h> +#include <tinyformat.h> +#include <util/chaintype.h> +#include <util/strencodings.h> #include <util/string.h> -#include <assert.h> +#include <cassert> +#include <cstdint> +#include <limits> +#include <stdexcept> +#include <vector> void ReadSigNetArgs(const ArgsManager& args, CChainParams::SigNetOptions& options) { @@ -24,9 +29,13 @@ void ReadSigNetArgs(const ArgsManager& args, CChainParams::SigNetOptions& option if (args.IsArgSet("-signetchallenge")) { const auto signet_challenge = args.GetArgs("-signetchallenge"); if (signet_challenge.size() != 1) { - throw std::runtime_error(strprintf("%s: -signetchallenge cannot be multiple values.", __func__)); + throw std::runtime_error("-signetchallenge cannot be multiple values."); } - options.challenge.emplace(ParseHex(signet_challenge[0])); + const auto val{TryParseHex<uint8_t>(signet_challenge[0])}; + if (!val) { + throw std::runtime_error(strprintf("-signetchallenge must be hex, not '%s'.", signet_challenge[0])); + } + options.challenge.emplace(*val); } } @@ -97,26 +106,29 @@ const CChainParams &Params() { return *globalChainParams; } -std::unique_ptr<const CChainParams> CreateChainParams(const ArgsManager& args, const std::string& chain) +std::unique_ptr<const CChainParams> CreateChainParams(const ArgsManager& args, const ChainType chain) { - if (chain == CBaseChainParams::MAIN) { + switch (chain) { + case ChainType::MAIN: return CChainParams::Main(); - } else if (chain == CBaseChainParams::TESTNET) { + case ChainType::TESTNET: return CChainParams::TestNet(); - } else if (chain == CBaseChainParams::SIGNET) { + case ChainType::SIGNET: { auto opts = CChainParams::SigNetOptions{}; ReadSigNetArgs(args, opts); return CChainParams::SigNet(opts); - } else if (chain == CBaseChainParams::REGTEST) { + } + case ChainType::REGTEST: { auto opts = CChainParams::RegTestOptions{}; ReadRegTestArgs(args, opts); return CChainParams::RegTest(opts); } - throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); + } + assert(false); } -void SelectParams(const std::string& network) +void SelectParams(const ChainType chain) { - SelectBaseParams(network); - globalChainParams = CreateChainParams(gArgs, network); + SelectBaseParams(chain); + globalChainParams = CreateChainParams(gArgs, chain); } diff --git a/src/chainparams.h b/src/chainparams.h index cb34d068e1..4743e022db 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -6,27 +6,16 @@ #ifndef BITCOIN_CHAINPARAMS_H #define BITCOIN_CHAINPARAMS_H -#include <kernel/chainparams.h> +#include <kernel/chainparams.h> // IWYU pragma: export -#include <chainparamsbase.h> -#include <consensus/params.h> -#include <netaddress.h> -#include <primitives/block.h> -#include <protocol.h> -#include <util/hash_type.h> - -#include <cstdint> #include <memory> -#include <string> -#include <unordered_map> -#include <vector> + +class ArgsManager; /** * Creates and returns a std::unique_ptr<CChainParams> of the chosen chain. - * @returns a CChainParams* of the chosen chain. - * @throws a std::runtime_error if the chain is not supported. */ -std::unique_ptr<const CChainParams> CreateChainParams(const ArgsManager& args, const std::string& chain); +std::unique_ptr<const CChainParams> CreateChainParams(const ArgsManager& args, const ChainType chain); /** * Return the currently selected parameters. This won't change after app @@ -35,9 +24,8 @@ std::unique_ptr<const CChainParams> CreateChainParams(const ArgsManager& args, c const CChainParams &Params(); /** - * Sets the params returned by Params() to those for the given chain name. - * @throws std::runtime_error when the chain is not supported. + * Sets the params returned by Params() to those for the given chain type. */ -void SelectParams(const std::string& chain); +void SelectParams(const ChainType chain); #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index eb7b31923d..8cbf9e85e0 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -7,14 +7,10 @@ #include <common/args.h> #include <tinyformat.h> +#include <util/chaintype.h> #include <assert.h> -const std::string CBaseChainParams::MAIN = "main"; -const std::string CBaseChainParams::TESTNET = "test"; -const std::string CBaseChainParams::SIGNET = "signet"; -const std::string CBaseChainParams::REGTEST = "regtest"; - void SetupChainParamsBaseOptions(ArgsManager& argsman) { argsman.AddArg("-chain=<chain>", "Use the chain <chain> (default: main). Allowed values: main, test, signet, regtest", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); @@ -40,22 +36,23 @@ const CBaseChainParams& BaseParams() * Port numbers for incoming Tor connections (8334, 18334, 38334, 18445) have * been chosen arbitrarily to keep ranges of used ports tight. */ -std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain) +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const ChainType chain) { - if (chain == CBaseChainParams::MAIN) { + switch (chain) { + case ChainType::MAIN: return std::make_unique<CBaseChainParams>("", 8332, 8334); - } else if (chain == CBaseChainParams::TESTNET) { + case ChainType::TESTNET: return std::make_unique<CBaseChainParams>("testnet3", 18332, 18334); - } else if (chain == CBaseChainParams::SIGNET) { + case ChainType::SIGNET: return std::make_unique<CBaseChainParams>("signet", 38332, 38334); - } else if (chain == CBaseChainParams::REGTEST) { + case ChainType::REGTEST: return std::make_unique<CBaseChainParams>("regtest", 18443, 18445); } - throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); + assert(false); } -void SelectBaseParams(const std::string& chain) +void SelectBaseParams(const ChainType chain) { globalChainBaseParams = CreateBaseChainParams(chain); - gArgs.SelectConfigNetwork(chain); + gArgs.SelectConfigNetwork(ChainTypeToString(chain)); } diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index d593cff722..ea933d1ca8 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CHAINPARAMSBASE_H #define BITCOIN_CHAINPARAMSBASE_H +#include <util/chaintype.h> + #include <memory> #include <string> @@ -17,14 +19,6 @@ class ArgsManager; class CBaseChainParams { public: - ///@{ - /** Chain name strings */ - static const std::string MAIN; - static const std::string TESTNET; - static const std::string SIGNET; - static const std::string REGTEST; - ///@} - const std::string& DataDir() const { return strDataDir; } uint16_t RPCPort() const { return m_rpc_port; } uint16_t OnionServiceTargetPort() const { return m_onion_service_target_port; } @@ -41,10 +35,8 @@ private: /** * Creates and returns a std::unique_ptr<CBaseChainParams> of the chosen chain. - * @returns a CBaseChainParams* of the chosen chain. - * @throws a std::runtime_error if the chain is not supported. */ -std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain); +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const ChainType chain); /** *Set the arguments for chainparams @@ -57,7 +49,7 @@ void SetupChainParamsBaseOptions(ArgsManager& argsman); */ const CBaseChainParams& BaseParams(); -/** Sets the params returned by Params() to those for the given network. */ -void SelectBaseParams(const std::string& chain); +/** Sets the params returned by Params() to those for the given chain. */ +void SelectBaseParams(const ChainType chain); #endif // BITCOIN_CHAINPARAMSBASE_H diff --git a/src/common/args.cpp b/src/common/args.cpp index d29b8648bf..643838399f 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -6,14 +6,15 @@ #include <common/args.h> #include <chainparamsbase.h> +#include <common/settings.h> #include <logging.h> #include <sync.h> #include <tinyformat.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/check.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/settings.h> #include <util/strencodings.h> #ifdef WIN32 @@ -33,6 +34,7 @@ #include <stdexcept> #include <string> #include <utility> +#include <variant> const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char * const BITCOIN_SETTINGS_FILENAME = "settings.json"; @@ -102,7 +104,7 @@ KeyInfo InterpretKey(std::string key) * @return parsed settings value if it is valid, otherwise nullopt accompanied * by a descriptive error string */ -std::optional<util::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, +std::optional<common::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, unsigned int flags, std::string& error) { // Return negated settings as false values. @@ -141,7 +143,7 @@ std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const if (m_network.empty()) return std::set<std::string> {}; // if it's okay to use the default section for this network, don't worry - if (m_network == CBaseChainParams::MAIN) return std::set<std::string> {}; + if (m_network == ChainTypeToString(ChainType::MAIN)) return std::set<std::string> {}; for (const auto& arg : m_network_only_args) { if (OnlyHasDefaultSectionSetting(m_settings, m_network, SettingName(arg))) { @@ -155,10 +157,10 @@ std::list<SectionInfo> ArgsManager::GetUnrecognizedSections() const { // Section names to be recognized in the config file. static const std::set<std::string> available_sections{ - CBaseChainParams::REGTEST, - CBaseChainParams::SIGNET, - CBaseChainParams::TESTNET, - CBaseChainParams::MAIN + ChainTypeToString(ChainType::REGTEST), + ChainTypeToString(ChainType::SIGNET), + ChainTypeToString(ChainType::TESTNET), + ChainTypeToString(ChainType::MAIN), }; LOCK(cs_args); @@ -236,15 +238,15 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin return false; } - std::optional<util::SettingsValue> value = InterpretValue(keyinfo, val ? &*val : nullptr, *flags, error); + std::optional<common::SettingsValue> value = InterpretValue(keyinfo, val ? &*val : nullptr, *flags, error); if (!value) return false; m_settings.command_line_options[keyinfo.name].push_back(*value); } // we do not allow -includeconf from command line, only -noincludeconf - if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { - const util::SettingsSpan values{*includes}; + if (auto* includes = common::FindKey(m_settings.command_line_options, "includeconf")) { + const common::SettingsSpan values{*includes}; // Range may be empty if -noincludeconf was passed if (!values.empty()) { error = "-includeconf cannot be used from commandline; -includeconf=" + values.begin()->write(); @@ -359,7 +361,7 @@ std::optional<const ArgsManager::Command> ArgsManager::GetCommand() const std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) const { std::vector<std::string> result; - for (const util::SettingsValue& value : GetSettingsList(strArg)) { + for (const common::SettingsValue& value : GetSettingsList(strArg)) { result.push_back(value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str()); } return result; @@ -406,7 +408,7 @@ bool ArgsManager::ReadSettingsFile(std::vector<std::string>* errors) LOCK(cs_args); m_settings.rw_settings.clear(); std::vector<std::string> read_errors; - if (!util::ReadSettings(path, m_settings.rw_settings, read_errors)) { + if (!common::ReadSettings(path, m_settings.rw_settings, read_errors)) { SaveErrors(read_errors, errors); return false; } @@ -428,7 +430,7 @@ bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors, bool backu LOCK(cs_args); std::vector<std::string> write_errors; - if (!util::WriteSettings(path_tmp, m_settings.rw_settings, write_errors)) { + if (!common::WriteSettings(path_tmp, m_settings.rw_settings, write_errors)) { SaveErrors(write_errors, errors); return false; } @@ -439,11 +441,11 @@ bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors, bool backu return true; } -util::SettingsValue ArgsManager::GetPersistentSetting(const std::string& name) const +common::SettingsValue ArgsManager::GetPersistentSetting(const std::string& name) const { LOCK(cs_args); - return util::GetSetting(m_settings, m_network, name, !UseDefaultSection("-" + name), - /*ignore_nonpersistent=*/true, /*get_chain_name=*/false); + return common::GetSetting(m_settings, m_network, name, !UseDefaultSection("-" + name), + /*ignore_nonpersistent=*/true, /*get_chain_type=*/false); } bool ArgsManager::IsArgNegated(const std::string& strArg) const @@ -458,11 +460,11 @@ std::string ArgsManager::GetArg(const std::string& strArg, const std::string& st std::optional<std::string> ArgsManager::GetArg(const std::string& strArg) const { - const util::SettingsValue value = GetSetting(strArg); + const common::SettingsValue value = GetSetting(strArg); return SettingToString(value); } -std::optional<std::string> SettingToString(const util::SettingsValue& value) +std::optional<std::string> SettingToString(const common::SettingsValue& value) { if (value.isNull()) return std::nullopt; if (value.isFalse()) return "0"; @@ -471,7 +473,7 @@ std::optional<std::string> SettingToString(const util::SettingsValue& value) return value.get_str(); } -std::string SettingToString(const util::SettingsValue& value, const std::string& strDefault) +std::string SettingToString(const common::SettingsValue& value, const std::string& strDefault) { return SettingToString(value).value_or(strDefault); } @@ -483,11 +485,11 @@ int64_t ArgsManager::GetIntArg(const std::string& strArg, int64_t nDefault) cons std::optional<int64_t> ArgsManager::GetIntArg(const std::string& strArg) const { - const util::SettingsValue value = GetSetting(strArg); + const common::SettingsValue value = GetSetting(strArg); return SettingToInt(value); } -std::optional<int64_t> SettingToInt(const util::SettingsValue& value) +std::optional<int64_t> SettingToInt(const common::SettingsValue& value) { if (value.isNull()) return std::nullopt; if (value.isFalse()) return 0; @@ -496,7 +498,7 @@ std::optional<int64_t> SettingToInt(const util::SettingsValue& value) return LocaleIndependentAtoi<int64_t>(value.get_str()); } -int64_t SettingToInt(const util::SettingsValue& value, int64_t nDefault) +int64_t SettingToInt(const common::SettingsValue& value, int64_t nDefault) { return SettingToInt(value).value_or(nDefault); } @@ -508,18 +510,18 @@ bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) const std::optional<bool> ArgsManager::GetBoolArg(const std::string& strArg) const { - const util::SettingsValue value = GetSetting(strArg); + const common::SettingsValue value = GetSetting(strArg); return SettingToBool(value); } -std::optional<bool> SettingToBool(const util::SettingsValue& value) +std::optional<bool> SettingToBool(const common::SettingsValue& value) { if (value.isNull()) return std::nullopt; if (value.isBool()) return value.get_bool(); return InterpretBool(value.get_str()); } -bool SettingToBool(const util::SettingsValue& value, bool fDefault) +bool SettingToBool(const common::SettingsValue& value, bool fDefault) { return SettingToBool(value).value_or(fDefault); } @@ -714,62 +716,77 @@ bool CheckDataDirOption(const ArgsManager& args) fs::path ArgsManager::GetConfigFilePath() const { - return GetConfigFile(*this, GetPathArg("-conf", BITCOIN_CONF_FILENAME)); + LOCK(cs_args); + return *Assert(m_config_path); +} + +ChainType ArgsManager::GetChainType() const +{ + std::variant<ChainType, std::string> arg = GetChainArg(); + if (auto* parsed = std::get_if<ChainType>(&arg)) return *parsed; + throw std::runtime_error(strprintf("Unknown chain %s.", std::get<std::string>(arg))); } -std::string ArgsManager::GetChainName() const +std::string ArgsManager::GetChainTypeString() const +{ + auto arg = GetChainArg(); + if (auto* parsed = std::get_if<ChainType>(&arg)) return ChainTypeToString(*parsed); + return std::get<std::string>(arg); +} + +std::variant<ChainType, std::string> ArgsManager::GetChainArg() const { auto get_net = [&](const std::string& arg) { LOCK(cs_args); - util::SettingsValue value = util::GetSetting(m_settings, /* section= */ "", SettingName(arg), + common::SettingsValue value = common::GetSetting(m_settings, /* section= */ "", SettingName(arg), /* ignore_default_section_config= */ false, /*ignore_nonpersistent=*/false, - /* get_chain_name= */ true); + /* get_chain_type= */ true); return value.isNull() ? false : value.isBool() ? value.get_bool() : InterpretBool(value.get_str()); }; const bool fRegTest = get_net("-regtest"); const bool fSigNet = get_net("-signet"); const bool fTestNet = get_net("-testnet"); - const bool is_chain_arg_set = IsArgSet("-chain"); + const auto chain_arg = GetArg("-chain"); - if ((int)is_chain_arg_set + (int)fRegTest + (int)fSigNet + (int)fTestNet > 1) { + if ((int)chain_arg.has_value() + (int)fRegTest + (int)fSigNet + (int)fTestNet > 1) { throw std::runtime_error("Invalid combination of -regtest, -signet, -testnet and -chain. Can use at most one."); } - if (fRegTest) - return CBaseChainParams::REGTEST; - if (fSigNet) { - return CBaseChainParams::SIGNET; + if (chain_arg) { + if (auto parsed = ChainTypeFromString(*chain_arg)) return *parsed; + // Not a known string, so return original string + return *chain_arg; } - if (fTestNet) - return CBaseChainParams::TESTNET; - - return GetArg("-chain", CBaseChainParams::MAIN); + if (fRegTest) return ChainType::REGTEST; + if (fSigNet) return ChainType::SIGNET; + if (fTestNet) return ChainType::TESTNET; + return ChainType::MAIN; } bool ArgsManager::UseDefaultSection(const std::string& arg) const { - return m_network == CBaseChainParams::MAIN || m_network_only_args.count(arg) == 0; + return m_network == ChainTypeToString(ChainType::MAIN) || m_network_only_args.count(arg) == 0; } -util::SettingsValue ArgsManager::GetSetting(const std::string& arg) const +common::SettingsValue ArgsManager::GetSetting(const std::string& arg) const { LOCK(cs_args); - return util::GetSetting( + return common::GetSetting( m_settings, m_network, SettingName(arg), !UseDefaultSection(arg), - /*ignore_nonpersistent=*/false, /*get_chain_name=*/false); + /*ignore_nonpersistent=*/false, /*get_chain_type=*/false); } -std::vector<util::SettingsValue> ArgsManager::GetSettingsList(const std::string& arg) const +std::vector<common::SettingsValue> ArgsManager::GetSettingsList(const std::string& arg) const { LOCK(cs_args); - return util::GetSettingsList(m_settings, m_network, SettingName(arg), !UseDefaultSection(arg)); + return common::GetSettingsList(m_settings, m_network, SettingName(arg), !UseDefaultSection(arg)); } void ArgsManager::logArgsPrefix( const std::string& prefix, const std::string& section, - const std::map<std::string, std::vector<util::SettingsValue>>& args) const + const std::map<std::string, std::vector<common::SettingsValue>>& args) const { std::string section_str = section.empty() ? "" : "[" + section + "] "; for (const auto& arg : args) { diff --git a/src/common/args.h b/src/common/args.h index 430c392e2b..ae3ed02bc7 100644 --- a/src/common/args.h +++ b/src/common/args.h @@ -5,10 +5,11 @@ #ifndef BITCOIN_COMMON_ARGS_H #define BITCOIN_COMMON_ARGS_H +#include <common/settings.h> #include <compat/compat.h> #include <sync.h> +#include <util/chaintype.h> #include <util/fs.h> -#include <util/settings.h> #include <iosfwd> #include <list> @@ -17,6 +18,7 @@ #include <set> #include <stdint.h> #include <string> +#include <variant> #include <vector> class ArgsManager; @@ -26,7 +28,6 @@ extern const char * const BITCOIN_SETTINGS_FILENAME; // Return true if -datadir option points to a valid directory or is not specified. bool CheckDataDirOption(const ArgsManager& args); -fs::path GetConfigFile(const ArgsManager& args, const fs::path& configuration_file_path); /** * Most paths passed as configuration arguments are treated as relative to @@ -74,7 +75,7 @@ struct KeyInfo { KeyInfo InterpretKey(std::string key); -std::optional<util::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, +std::optional<common::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, unsigned int flags, std::string& error); struct SectionInfo { @@ -83,14 +84,14 @@ struct SectionInfo { int m_line; }; -std::string SettingToString(const util::SettingsValue&, const std::string&); -std::optional<std::string> SettingToString(const util::SettingsValue&); +std::string SettingToString(const common::SettingsValue&, const std::string&); +std::optional<std::string> SettingToString(const common::SettingsValue&); -int64_t SettingToInt(const util::SettingsValue&, int64_t); -std::optional<int64_t> SettingToInt(const util::SettingsValue&); +int64_t SettingToInt(const common::SettingsValue&, int64_t); +std::optional<int64_t> SettingToInt(const common::SettingsValue&); -bool SettingToBool(const util::SettingsValue&, bool); -std::optional<bool> SettingToBool(const util::SettingsValue&); +bool SettingToBool(const common::SettingsValue&, bool); +std::optional<bool> SettingToBool(const common::SettingsValue&); class ArgsManager { @@ -129,13 +130,14 @@ protected: }; mutable RecursiveMutex cs_args; - util::Settings m_settings GUARDED_BY(cs_args); + common::Settings m_settings GUARDED_BY(cs_args); std::vector<std::string> m_command GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); bool m_accept_any_command GUARDED_BY(cs_args){true}; std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); + std::optional<fs::path> m_config_path GUARDED_BY(cs_args); mutable fs::path m_cached_blocks_path GUARDED_BY(cs_args); mutable fs::path m_cached_datadir_path GUARDED_BY(cs_args); mutable fs::path m_cached_network_datadir_path GUARDED_BY(cs_args); @@ -157,12 +159,12 @@ protected: * false if "-nosetting" argument was passed, and a string if a "-setting=value" * argument was passed. */ - util::SettingsValue GetSetting(const std::string& arg) const; + common::SettingsValue GetSetting(const std::string& arg) const; /** * Get list of setting values. */ - std::vector<util::SettingsValue> GetSettingsList(const std::string& arg) const; + std::vector<common::SettingsValue> GetSettingsList(const std::string& arg) const; ArgsManager(); ~ArgsManager(); @@ -323,10 +325,18 @@ protected: void ForceSetArg(const std::string& strArg, const std::string& strValue); /** - * Returns the appropriate chain name from the program arguments. - * @return CBaseChainParams::MAIN by default; raises runtime error if an invalid combination is given. + * Returns the appropriate chain type from the program arguments. + * @return ChainType::MAIN by default; raises runtime error if an invalid + * combination, or unknown chain is given. */ - std::string GetChainName() const; + ChainType GetChainType() const; + + /** + * Returns the appropriate chain type string from the program arguments. + * @return ChainType::MAIN string by default; raises runtime error if an + * invalid combination is given. + */ + std::string GetChainTypeString() const; /** * Add argument @@ -384,7 +394,7 @@ protected: * Get current setting from config file or read/write settings file, * ignoring nonpersistent command line or forced settings values. */ - util::SettingsValue GetPersistentSetting(const std::string& name) const; + common::SettingsValue GetPersistentSetting(const std::string& name) const; /** * Access settings with lock held. @@ -411,11 +421,19 @@ private: */ const fs::path& GetDataDir(bool net_specific) const; + /** + * Return -regtest/-signet/-testnet/-chain= setting as a ChainType enum if a + * recognized chain type was set, or as a string if an unrecognized chain + * name was set. Raise an exception if an invalid combination of flags was + * provided. + */ + std::variant<ChainType, std::string> GetChainArg() const; + // Helper function for LogArgs(). void logArgsPrefix( const std::string& prefix, const std::string& section, - const std::map<std::string, std::vector<util::SettingsValue>>& args) const; + const std::map<std::string, std::vector<common::SettingsValue>>& args) const; }; extern ArgsManager gArgs; diff --git a/src/common/config.cpp b/src/common/config.cpp index 747503ad2a..1c85273f69 100644 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -4,12 +4,13 @@ #include <common/args.h> +#include <common/settings.h> #include <logging.h> #include <sync.h> #include <tinyformat.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/fs.h> -#include <util/settings.h> #include <util/string.h> #include <algorithm> @@ -26,11 +27,6 @@ #include <utility> #include <vector> -fs::path GetConfigFile(const ArgsManager& args, const fs::path& configuration_file_path) -{ - return AbsPathForConfigVal(args, configuration_file_path, /*net_specific=*/false); -} - static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) { std::string str, prefix; @@ -102,7 +98,7 @@ bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& file std::optional<unsigned int> flags = GetArgFlags('-' + key.name); if (!IsConfSupported(key, error)) return false; if (flags) { - std::optional<util::SettingsValue> value = InterpretValue(key, &option.second, *flags, error); + std::optional<common::SettingsValue> value = InterpretValue(key, &option.second, *flags, error); if (!value) { return false; } @@ -125,6 +121,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) LOCK(cs_args); m_settings.ro_config.clear(); m_config_sections.clear(); + m_config_path = AbsPathForConfigVal(*this, GetPathArg("-conf", BITCOIN_CONF_FILENAME), /*net_specific=*/false); } const auto conf_path{GetConfigFilePath()}; @@ -145,22 +142,22 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) bool use_conf_file{true}; { LOCK(cs_args); - if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { + if (auto* includes = common::FindKey(m_settings.command_line_options, "includeconf")) { // ParseParameters() fails if a non-negated -includeconf is passed on the command-line - assert(util::SettingsSpan(*includes).last_negated()); + assert(common::SettingsSpan(*includes).last_negated()); use_conf_file = false; } } if (use_conf_file) { - std::string chain_id = GetChainName(); + std::string chain_id = GetChainTypeString(); std::vector<std::string> conf_file_names; auto add_includes = [&](const std::string& network, size_t skip = 0) { size_t num_values = 0; LOCK(cs_args); - if (auto* section = util::FindKey(m_settings.ro_config, network)) { - if (auto* values = util::FindKey(*section, "includeconf")) { - for (size_t i = std::max(skip, util::SettingsSpan(*values).negated()); i < values->size(); ++i) { + if (auto* section = common::FindKey(m_settings.ro_config, network)) { + if (auto* values = common::FindKey(*section, "includeconf")) { + for (size_t i = std::max(skip, common::SettingsSpan(*values).negated()); i < values->size(); ++i) { conf_file_names.push_back((*values)[i].get_str()); } num_values = values->size(); @@ -175,7 +172,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) const size_t default_includes = add_includes({}); for (const std::string& conf_file_name : conf_file_names) { - std::ifstream conf_file_stream{GetConfigFile(*this, fs::PathFromString(conf_file_name))}; + std::ifstream conf_file_stream{AbsPathForConfigVal(*this, fs::PathFromString(conf_file_name), /*net_specific=*/false)}; if (conf_file_stream.good()) { if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; @@ -191,7 +188,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) conf_file_names.clear(); add_includes(chain_id, /* skip= */ chain_includes); add_includes({}, /* skip= */ default_includes); - std::string chain_id_final = GetChainName(); + std::string chain_id_final = GetChainTypeString(); if (chain_id_final != chain_id) { // Also warn about recursive includeconf for the chain that was specified in one of the includeconfs add_includes(chain_id_final); diff --git a/src/common/init.cpp b/src/common/init.cpp index 6ffa44847a..412d73aec7 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -5,6 +5,7 @@ #include <chainparams.h> #include <common/args.h> #include <common/init.h> +#include <logging.h> #include <tinyformat.h> #include <util/fs.h> #include <util/translation.h> @@ -20,13 +21,26 @@ std::optional<ConfigError> InitConfig(ArgsManager& args, SettingsAbortFn setting if (!CheckDataDirOption(args)) { return ConfigError{ConfigStatus::FAILED, strprintf(_("Specified data directory \"%s\" does not exist."), args.GetArg("-datadir", ""))}; } + + // Record original datadir and config paths before parsing the config + // file. It is possible for the config file to contain a datadir= line + // that changes the datadir path after it is parsed. This is useful for + // CLI tools to let them use a different data storage location without + // needing to pass it every time on the command line. (It is not + // possible for the config file to cause another configuration to be + // used, though. Specifying a conf= option in the config file causes a + // parse error, and specifying a datadir= location containing another + // bitcoin.conf file just ignores the other file.) + const fs::path orig_datadir_path{args.GetDataDirBase()}; + const fs::path orig_config_path{AbsPathForConfigVal(args, args.GetPathArg("-conf", BITCOIN_CONF_FILENAME), /*net_specific=*/false)}; + std::string error; if (!args.ReadConfigFiles(error, true)) { return ConfigError{ConfigStatus::FAILED, strprintf(_("Error reading configuration file: %s"), error)}; } // Check for chain settings (Params() calls are only valid after this clause) - SelectParams(args.GetChainName()); + SelectParams(args.GetChainType()); // Create datadir if it does not exist. const auto base_path{args.GetDataDirBase()}; @@ -48,6 +62,32 @@ std::optional<ConfigError> InitConfig(ArgsManager& args, SettingsAbortFn setting fs::create_directories(net_path / "wallets"); } + // Show an error or warning if there is a bitcoin.conf file in the + // datadir that is being ignored. + const fs::path base_config_path = base_path / BITCOIN_CONF_FILENAME; + if (fs::exists(base_config_path) && !fs::equivalent(orig_config_path, base_config_path)) { + const std::string cli_config_path = args.GetArg("-conf", ""); + const std::string config_source = cli_config_path.empty() + ? strprintf("data directory %s", fs::quoted(fs::PathToString(orig_datadir_path))) + : strprintf("command line argument %s", fs::quoted("-conf=" + cli_config_path)); + const std::string error = strprintf( + "Data directory %1$s contains a %2$s file which is ignored, because a different configuration file " + "%3$s from %4$s is being used instead. Possible ways to address this would be to:\n" + "- Delete or rename the %2$s file in data directory %1$s.\n" + "- Change datadir= or conf= options to specify one configuration file, not two, and use " + "includeconf= to include any other configuration files.\n" + "- Set allowignoredconf=1 option to treat this condition as a warning, not an error.", + fs::quoted(fs::PathToString(base_path)), + fs::quoted(BITCOIN_CONF_FILENAME), + fs::quoted(fs::PathToString(orig_config_path)), + config_source); + if (args.GetBoolArg("-allowignoredconf", false)) { + LogPrintf("Warning: %s\n", error); + } else { + return ConfigError{ConfigStatus::FAILED, Untranslated(error)}; + } + } + // Create settings.json if -nosettings was not specified. if (args.GetSettingsPath()) { std::vector<std::string> details; diff --git a/src/util/settings.cpp b/src/common/settings.cpp index bb257c0584..9187f242eb 100644 --- a/src/util/settings.cpp +++ b/src/common/settings.cpp @@ -2,18 +2,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/fs.h> -#include <util/settings.h> +#include <common/settings.h> #include <tinyformat.h> #include <univalue.h> +#include <util/fs.h> +#include <algorithm> #include <fstream> +#include <iterator> #include <map> #include <string> +#include <utility> #include <vector> -namespace util { +namespace common { namespace { enum class Source { @@ -130,7 +133,7 @@ SettingsValue GetSetting(const Settings& settings, const std::string& name, bool ignore_default_section_config, bool ignore_nonpersistent, - bool get_chain_name) + bool get_chain_type) { SettingsValue result; bool done = false; // Done merging any more settings sources. @@ -145,17 +148,17 @@ SettingsValue GetSetting(const Settings& settings, // assigned value instead of last. In general, later settings take // precedence over early settings, but for backwards compatibility in // the config file the precedence is reversed for all settings except - // chain name settings. + // chain type settings. const bool reverse_precedence = (source == Source::CONFIG_FILE_NETWORK_SECTION || source == Source::CONFIG_FILE_DEFAULT_SECTION) && - !get_chain_name; + !get_chain_type; // Weird behavior preserved for backwards compatibility: Negated // -regtest and -testnet arguments which you would expect to override // values set in the configuration file are currently accepted but // silently ignored. It would be better to apply these just like other // negated values, or at least warn they are ignored. - const bool skip_negated_command_line = get_chain_name; + const bool skip_negated_command_line = get_chain_type; if (done) return; @@ -255,4 +258,4 @@ size_t SettingsSpan::negated() const return 0; } -} // namespace util +} // namespace common diff --git a/src/util/settings.h b/src/common/settings.h index 27e87a4751..0e9d376e23 100644 --- a/src/util/settings.h +++ b/src/common/settings.h @@ -2,18 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UTIL_SETTINGS_H -#define BITCOIN_UTIL_SETTINGS_H +#ifndef BITCOIN_COMMON_SETTINGS_H +#define BITCOIN_COMMON_SETTINGS_H #include <util/fs.h> +#include <cstddef> #include <map> #include <string> #include <vector> class UniValue; -namespace util { +namespace common { //! Settings value type (string/integer/boolean/null variant). //! @@ -60,14 +61,14 @@ bool WriteSettings(const fs::path& path, //! command line). Only return settings in the //! read-only config and read-write settings //! files. -//! @param get_chain_name - enable special backwards compatible behavior -//! for GetChainName +//! @param get_chain_type - enable special backwards compatible behavior +//! for GetChainType SettingsValue GetSetting(const Settings& settings, const std::string& section, const std::string& name, bool ignore_default_section_config, bool ignore_nonpersistent, - bool get_chain_name); + bool get_chain_type); //! Get combined setting value similar to GetSetting(), except if setting was //! specified multiple times, return a list of all the values specified. @@ -109,6 +110,6 @@ auto FindKey(Map&& map, Key&& key) -> decltype(&map.at(key)) return it == map.end() ? nullptr : &it->second; } -} // namespace util +} // namespace common -#endif // BITCOIN_UTIL_SETTINGS_H +#endif // BITCOIN_COMMON_SETTINGS_H diff --git a/src/util/system.cpp b/src/common/system.cpp index 598e6adb88..1d1c5fa56a 100644 --- a/src/util/system.cpp +++ b/src/common/system.cpp @@ -3,20 +3,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/system.h> +#include <common/system.h> #include <logging.h> #include <util/string.h> -#include <util/syserror.h> #include <util/time.h> -#if (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) -#include <pthread.h> -#include <pthread_np.h> -#endif - #ifndef WIN32 -#include <sched.h> #include <sys/stat.h> #else #include <codecvt> @@ -112,14 +105,3 @@ int64_t GetStartupTime() { return nStartupTime; } - -void ScheduleBatchPriority() -{ -#ifdef SCHED_BATCH - const static sched_param param{}; - const int rc = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); - if (rc != 0) { - LogPrintf("Failed to pthread_setschedparam: %s\n", SysErrorString(rc)); - } -#endif -} diff --git a/src/common/system.h b/src/common/system.h new file mode 100644 index 0000000000..40206aaa01 --- /dev/null +++ b/src/common/system.h @@ -0,0 +1,38 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMMON_SYSTEM_H +#define BITCOIN_COMMON_SYSTEM_H + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <compat/assumptions.h> +#include <compat/compat.h> + +#include <set> +#include <stdint.h> +#include <string> + +// Application startup time (used for uptime calculation) +int64_t GetStartupTime(); + +void SetupEnvironment(); +bool SetupNetworking(); +#ifndef WIN32 +std::string ShellEscape(const std::string& arg); +#endif +#if HAVE_SYSTEM +void runCommand(const std::string& strCommand); +#endif + +/** + * Return the number of cores available on the current system. + * @note This does count virtual cores, such as those provided by HyperThreading. + */ +int GetNumCores(); + +#endif // BITCOIN_COMMON_SYSTEM_H diff --git a/src/consensus/validation.h b/src/consensus/validation.h index ad8ee676b2..d5bf08cd61 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -145,7 +145,7 @@ class BlockValidationState : public ValidationState<BlockValidationResult> {}; // using only serialization with and without witness data. As witness_size // is equal to total_size - stripped_size, this formula is identical to: // weight = (stripped_size * 3) + total_size. -static inline int64_t GetTransactionWeight(const CTransaction& tx) +static inline int32_t GetTransactionWeight(const CTransaction& tx) { return ::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(tx, PROTOCOL_VERSION); } diff --git a/src/core_write.cpp b/src/core_write.cpp index b0e3b0b3c4..54ca306f60 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -4,6 +4,7 @@ #include <core_io.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/validation.h> @@ -17,7 +18,6 @@ #include <univalue.h> #include <util/check.h> #include <util/strencodings.h> -#include <util/system.h> #include <map> #include <string> diff --git a/src/crypto/sha256_avx2.cpp b/src/crypto/sha256_avx2.cpp index 624bdb42e4..df8cb7a6c9 100644 --- a/src/crypto/sha256_avx2.cpp +++ b/src/crypto/sha256_avx2.cpp @@ -7,6 +7,7 @@ #include <stdint.h> #include <immintrin.h> +#include <attributes.h> #include <crypto/common.h> namespace sha256d64_avx2 { @@ -36,7 +37,7 @@ __m256i inline sigma0(__m256i x) { return Xor(Or(ShR(x, 7), ShL(x, 25)), Or(ShR( __m256i inline sigma1(__m256i x) { return Xor(Or(ShR(x, 17), ShL(x, 15)), Or(ShR(x, 19), ShL(x, 13)), ShR(x, 10)); } /** One round of SHA-256. */ -void inline __attribute__((always_inline)) Round(__m256i a, __m256i b, __m256i c, __m256i& d, __m256i e, __m256i f, __m256i g, __m256i& h, __m256i k) +void ALWAYS_INLINE Round(__m256i a, __m256i b, __m256i c, __m256i& d, __m256i e, __m256i f, __m256i g, __m256i& h, __m256i k) { __m256i t1 = Add(h, Sigma1(e), Ch(e, f, g), k); __m256i t2 = Add(Sigma0(a), Maj(a, b, c)); diff --git a/src/crypto/sha256_sse41.cpp b/src/crypto/sha256_sse41.cpp index 4eaf7d7b18..d041fdfefc 100644 --- a/src/crypto/sha256_sse41.cpp +++ b/src/crypto/sha256_sse41.cpp @@ -7,6 +7,7 @@ #include <stdint.h> #include <immintrin.h> +#include <attributes.h> #include <crypto/common.h> namespace sha256d64_sse41 { @@ -36,7 +37,7 @@ __m128i inline sigma0(__m128i x) { return Xor(Or(ShR(x, 7), ShL(x, 25)), Or(ShR( __m128i inline sigma1(__m128i x) { return Xor(Or(ShR(x, 17), ShL(x, 15)), Or(ShR(x, 19), ShL(x, 13)), ShR(x, 10)); } /** One round of SHA-256. */ -void inline __attribute__((always_inline)) Round(__m128i a, __m128i b, __m128i c, __m128i& d, __m128i e, __m128i f, __m128i g, __m128i& h, __m128i k) +void ALWAYS_INLINE Round(__m128i a, __m128i b, __m128i c, __m128i& d, __m128i e, __m128i f, __m128i g, __m128i& h, __m128i k) { __m128i t1 = Add(h, Sigma1(e), Ch(e, f, g), k); __m128i t2 = Add(Sigma0(a), Maj(a, b, c)); diff --git a/src/crypto/sha256_x86_shani.cpp b/src/crypto/sha256_x86_shani.cpp index e3143a55c2..79871bfcc1 100644 --- a/src/crypto/sha256_x86_shani.cpp +++ b/src/crypto/sha256_x86_shani.cpp @@ -11,43 +11,45 @@ #include <stdint.h> #include <immintrin.h> +#include <attributes.h> + namespace { alignas(__m128i) const uint8_t MASK[16] = {0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04, 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c}; alignas(__m128i) const uint8_t INIT0[16] = {0x8c, 0x68, 0x05, 0x9b, 0x7f, 0x52, 0x0e, 0x51, 0x85, 0xae, 0x67, 0xbb, 0x67, 0xe6, 0x09, 0x6a}; alignas(__m128i) const uint8_t INIT1[16] = {0x19, 0xcd, 0xe0, 0x5b, 0xab, 0xd9, 0x83, 0x1f, 0x3a, 0xf5, 0x4f, 0xa5, 0x72, 0xf3, 0x6e, 0x3c}; -void inline __attribute__((always_inline)) QuadRound(__m128i& state0, __m128i& state1, uint64_t k1, uint64_t k0) +void ALWAYS_INLINE QuadRound(__m128i& state0, __m128i& state1, uint64_t k1, uint64_t k0) { const __m128i msg = _mm_set_epi64x(k1, k0); state1 = _mm_sha256rnds2_epu32(state1, state0, msg); state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e)); } -void inline __attribute__((always_inline)) QuadRound(__m128i& state0, __m128i& state1, __m128i m, uint64_t k1, uint64_t k0) +void ALWAYS_INLINE QuadRound(__m128i& state0, __m128i& state1, __m128i m, uint64_t k1, uint64_t k0) { const __m128i msg = _mm_add_epi32(m, _mm_set_epi64x(k1, k0)); state1 = _mm_sha256rnds2_epu32(state1, state0, msg); state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e)); } -void inline __attribute__((always_inline)) ShiftMessageA(__m128i& m0, __m128i m1) +void ALWAYS_INLINE ShiftMessageA(__m128i& m0, __m128i m1) { m0 = _mm_sha256msg1_epu32(m0, m1); } -void inline __attribute__((always_inline)) ShiftMessageC(__m128i& m0, __m128i m1, __m128i& m2) +void ALWAYS_INLINE ShiftMessageC(__m128i& m0, __m128i m1, __m128i& m2) { m2 = _mm_sha256msg2_epu32(_mm_add_epi32(m2, _mm_alignr_epi8(m1, m0, 4)), m1); } -void inline __attribute__((always_inline)) ShiftMessageB(__m128i& m0, __m128i m1, __m128i& m2) +void ALWAYS_INLINE ShiftMessageB(__m128i& m0, __m128i m1, __m128i& m2) { ShiftMessageC(m0, m1, m2); ShiftMessageA(m0, m1); } -void inline __attribute__((always_inline)) Shuffle(__m128i& s0, __m128i& s1) +void ALWAYS_INLINE Shuffle(__m128i& s0, __m128i& s1) { const __m128i t1 = _mm_shuffle_epi32(s0, 0xB1); const __m128i t2 = _mm_shuffle_epi32(s1, 0x1B); @@ -55,7 +57,7 @@ void inline __attribute__((always_inline)) Shuffle(__m128i& s0, __m128i& s1) s1 = _mm_blend_epi16(t2, t1, 0xF0); } -void inline __attribute__((always_inline)) Unshuffle(__m128i& s0, __m128i& s1) +void ALWAYS_INLINE Unshuffle(__m128i& s0, __m128i& s1) { const __m128i t1 = _mm_shuffle_epi32(s0, 0x1B); const __m128i t2 = _mm_shuffle_epi32(s1, 0xB1); @@ -63,12 +65,12 @@ void inline __attribute__((always_inline)) Unshuffle(__m128i& s0, __m128i& s1) s1 = _mm_alignr_epi8(t2, t1, 0x08); } -__m128i inline __attribute__((always_inline)) Load(const unsigned char* in) +__m128i ALWAYS_INLINE Load(const unsigned char* in) { return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)in), _mm_load_si128((const __m128i*)MASK)); } -void inline __attribute__((always_inline)) Save(unsigned char* out, __m128i s) +void ALWAYS_INLINE Save(unsigned char* out, __m128i s) { _mm_storeu_si128((__m128i*)out, _mm_shuffle_epi8(s, _mm_load_si128((const __m128i*)MASK))); } diff --git a/src/external_signer.cpp b/src/external_signer.cpp index 5524b943f4..6b1e1f0241 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -30,7 +30,7 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS } for (const UniValue& signer : result.getValues()) { // Check for error - const UniValue& error = find_value(signer, "error"); + const UniValue& error = signer.find_value("error"); if (!error.isNull()) { if (!error.isStr()) { throw std::runtime_error(strprintf("'%s' error", command)); @@ -38,11 +38,11 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS throw std::runtime_error(strprintf("'%s' error: %s", command, error.getValStr())); } // Check if fingerprint is present - const UniValue& fingerprint = find_value(signer, "fingerprint"); + const UniValue& fingerprint = signer.find_value("fingerprint"); if (fingerprint.isNull()) { throw std::runtime_error(strprintf("'%s' received invalid response, missing signer fingerprint", command)); } - const std::string fingerprintStr = fingerprint.get_str(); + const std::string& fingerprintStr{fingerprint.get_str()}; // Skip duplicate signer bool duplicate = false; for (const ExternalSigner& signer : signers) { @@ -50,7 +50,7 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS } if (duplicate) break; std::string name; - const UniValue& model_field = find_value(signer, "model"); + const UniValue& model_field = signer.find_value("model"); if (model_field.isStr() && model_field.getValStr() != "") { name += model_field.getValStr(); } @@ -97,19 +97,19 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str const UniValue signer_result = RunCommandParseJSON(command, stdinStr); - if (find_value(signer_result, "error").isStr()) { - error = find_value(signer_result, "error").get_str(); + if (signer_result.find_value("error").isStr()) { + error = signer_result.find_value("error").get_str(); return false; } - if (!find_value(signer_result, "psbt").isStr()) { + if (!signer_result.find_value("psbt").isStr()) { error = "Unexpected result from signer"; return false; } PartiallySignedTransaction signer_psbtx; std::string signer_psbt_error; - if (!DecodeBase64PSBT(signer_psbtx, find_value(signer_result, "psbt").get_str(), signer_psbt_error)) { + if (!DecodeBase64PSBT(signer_psbtx, signer_result.find_value("psbt").get_str(), signer_psbt_error)) { error = strprintf("TX decode failed %s", signer_psbt_error); return false; } diff --git a/src/external_signer.h b/src/external_signer.h index 90f07478e3..81a601811a 100644 --- a/src/external_signer.h +++ b/src/external_signer.h @@ -5,8 +5,8 @@ #ifndef BITCOIN_EXTERNAL_SIGNER_H #define BITCOIN_EXTERNAL_SIGNER_H +#include <common/system.h> #include <univalue.h> -#include <util/system.h> #include <string> #include <vector> diff --git a/src/httprpc.cpp b/src/httprpc.cpp index bf3fa6298d..661406e122 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -76,7 +76,7 @@ static void JSONErrorReply(HTTPRequest* req, const UniValue& objError, const Uni { // Send error reply from json-rpc error object int nStatus = HTTP_INTERNAL_SERVER_ERROR; - int code = find_value(objError, "code").getInt<int>(); + int code = objError.find_value("code").getInt<int>(); if (code == RPC_INVALID_REQUEST) nStatus = HTTP_BAD_REQUEST; @@ -213,7 +213,7 @@ static bool HTTPReq_JSONRPC(const std::any& context, HTTPRequest* req) } else { const UniValue& request = valRequest[reqIdx].get_obj(); // Parse method - std::string strMethod = find_value(request, "method").get_str(); + std::string strMethod = request.find_value("method").get_str(); if (!g_rpc_whitelist[jreq.authUser].count(strMethod)) { LogPrintf("RPC User %s not allowed to call method %s\n", jreq.authUser, strMethod); req->WriteReply(HTTP_FORBIDDEN); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 4f8a2b4d8d..128c4e3c56 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -170,12 +170,8 @@ static bool ClientAllowed(const CNetAddr& netaddr) static bool InitHTTPAllowList() { rpc_allow_subnets.clear(); - CNetAddr localv4; - CNetAddr localv6; - LookupHost("127.0.0.1", localv4, false); - LookupHost("::1", localv6, false); - rpc_allow_subnets.push_back(CSubNet(localv4, 8)); // always allow IPv4 local subnet - rpc_allow_subnets.push_back(CSubNet(localv6)); // always allow IPv6 localhost + rpc_allow_subnets.push_back(CSubNet{LookupHost("127.0.0.1", false).value(), 8}); // always allow IPv4 local subnet + rpc_allow_subnets.push_back(CSubNet{LookupHost("::1", false).value()}); // always allow IPv6 localhost for (const std::string& strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; LookupSubNet(strAllow, subnet); @@ -338,8 +334,8 @@ static bool HTTPBindAddresses(struct evhttp* http) LogPrintf("Binding RPC on address %s port %i\n", i->first, i->second); evhttp_bound_socket *bind_handle = evhttp_bind_socket_with_handle(http, i->first.empty() ? nullptr : i->first.c_str(), i->second); if (bind_handle) { - CNetAddr addr; - if (i->first.empty() || (LookupHost(i->first, addr, false) && addr.IsBindAny())) { + const std::optional<CNetAddr> addr{LookupHost(i->first, false)}; + if (i->first.empty() || (addr.has_value() && addr->IsBindAny())) { LogPrintf("WARNING: the RPC server is not safe to expose to untrusted networks such as the public internet\n"); } boundSockets.push_back(bind_handle); diff --git a/src/i2p.cpp b/src/i2p.cpp index c67b6ed185..f03e375adf 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -336,7 +336,7 @@ void Session::GenerateAndSavePrivateKey(const Sock& sock) { DestGenerate(sock); - // umask is set to 0077 in util/system.cpp, which is ok. + // umask is set to 0077 in common/system.cpp, which is ok. if (!WriteBinaryFile(m_private_key_file, std::string(m_private_key.begin(), m_private_key.end()))) { throw std::runtime_error( diff --git a/src/index/base.cpp b/src/index/base.cpp index 237c8e8be0..a713be3480 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -23,7 +23,7 @@ #include <string> #include <utility> -using node::ReadBlockFromDisk; +using node::g_indexes_ready_to_sync; constexpr uint8_t DB_BEST_BLOCK{'B'}; @@ -33,11 +33,7 @@ constexpr auto SYNC_LOCATOR_WRITE_INTERVAL{30s}; template <typename... Args> static void FatalError(const char* fmt, const Args&... args) { - std::string strMessage = tfm::format(fmt, args...); - SetMiscWarning(Untranslated(strMessage)); - LogPrintf("*** %s\n", strMessage); - InitError(_("A fatal internal error occurred, see debug.log for details")); - StartShutdown(); + AbortNode(tfm::format(fmt, args...)); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) @@ -94,16 +90,21 @@ bool BaseIndex::Init() if (locator.IsNull()) { SetBestBlockIndex(nullptr); } else { - SetBestBlockIndex(m_chainstate->FindForkInGlobalIndex(locator)); + // Setting the best block to the locator's top block. If it is not part of the + // best chain, we will rewind to the fork point during index sync + const CBlockIndex* locator_index{m_chainstate->m_blockman.LookupBlockIndex(locator.vHave.at(0))}; + if (!locator_index) { + return InitError(strprintf(Untranslated("%s: best block of the index not found. Please rebuild the index."), GetName())); + } + SetBestBlockIndex(locator_index); } - // Note: this will latch to true immediately if the user starts up with an empty - // datadir and an index enabled. If this is the case, indexation will happen solely - // via `BlockConnected` signals until, possibly, the next restart. - m_synced = m_best_block_index.load() == active_chain.Tip(); - if (!m_synced) { + // Skip pruning check if indexes are not ready to sync (because reindex-chainstate has wiped the chain). + const CBlockIndex* start_block = m_best_block_index.load(); + bool synced = start_block == active_chain.Tip(); + if (!synced && g_indexes_ready_to_sync) { bool prune_violation = false; - if (!m_best_block_index) { + if (!start_block) { // index is not built yet // make sure we have all block data back to the genesis prune_violation = m_chainstate->m_blockman.GetFirstStoredBlock(*active_chain.Tip()) != active_chain.Genesis(); @@ -111,7 +112,7 @@ bool BaseIndex::Init() // in case the index has a best block set and is not fully synced // check if we have the required blocks to continue building the index else { - const CBlockIndex* block_to_test = m_best_block_index.load(); + const CBlockIndex* block_to_test = start_block; if (!active_chain.Contains(block_to_test)) { // if the bestblock is not part of the mainchain, find the fork // and make sure we have all data down to the fork @@ -135,6 +136,16 @@ bool BaseIndex::Init() return InitError(strprintf(Untranslated("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"), GetName())); } } + + // Child init + if (!CustomInit(start_block ? std::make_optional(interfaces::BlockKey{start_block->GetBlockHash(), start_block->nHeight}) : std::nullopt)) { + return false; + } + + // Note: this will latch to true immediately if the user starts up with an empty + // datadir and an index enabled. If this is the case, indexation will happen solely + // via `BlockConnected` signals until, possibly, the next restart. + m_synced = synced; return true; } @@ -157,10 +168,14 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& void BaseIndex::ThreadSync() { SetSyscallSandboxPolicy(SyscallSandboxPolicy::TX_INDEX); + // Wait for a possible reindex-chainstate to finish until continuing + // with the index sync + while (!g_indexes_ready_to_sync) { + if (!m_interrupt.sleep_for(std::chrono::milliseconds(500))) return; + } + const CBlockIndex* pindex = m_best_block_index.load(); if (!m_synced) { - auto& consensus_params = Params().GetConsensus(); - std::chrono::steady_clock::time_point last_log_time{0s}; std::chrono::steady_clock::time_point last_locator_write_time{0s}; while (true) { @@ -207,7 +222,7 @@ void BaseIndex::ThreadSync() CBlock block; interfaces::BlockInfo block_info = kernel::MakeBlockInfo(pindex); - if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + if (!m_chainstate->m_blockman.ReadBlockFromDisk(block, *pindex)) { FatalError("%s: Failed to read block %s from disk", __func__, pindex->GetBlockHash().ToString()); return; @@ -396,11 +411,6 @@ bool BaseIndex::Start() RegisterValidationInterface(this); if (!Init()) return false; - const CBlockIndex* index = m_best_block_index.load(); - if (!CustomInit(index ? std::make_optional(interfaces::BlockKey{index->GetBlockHash(), index->nHeight}) : std::nullopt)) { - return false; - } - m_thread_sync = std::thread(&util::TraceThread, GetName(), [this] { ThreadSync(); }); return true; } diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index e6300ce3dd..a860b3a94d 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -12,8 +12,6 @@ #include <util/fs_helpers.h> #include <validation.h> -using node::UndoReadFromDisk; - /* The index database stores three items for each block: the disk location of the encoded filter, * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by * height, and those belonging to blocks that have been reorganized out of the active chain are @@ -223,7 +221,7 @@ bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block) // pindex variable gives indexing code access to node internals. It // will be removed in upcoming commit const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash)); - if (!UndoReadFromDisk(block_undo, pindex)) { + if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 5bc6ad3d31..d80885f842 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -19,9 +19,6 @@ using kernel::CCoinsStats; using kernel::GetBogoSize; using kernel::TxOutSer; -using node::ReadBlockFromDisk; -using node::UndoReadFromDisk; - static constexpr uint8_t DB_BLOCK_HASH{'s'}; static constexpr uint8_t DB_BLOCK_HEIGHT{'t'}; static constexpr uint8_t DB_MUHASH{'M'}; @@ -125,7 +122,7 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) // pindex variable gives indexing code access to node internals. It // will be removed in upcoming commit const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash)); - if (!UndoReadFromDisk(block_undo, pindex)) { + if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } @@ -282,12 +279,11 @@ bool CoinStatsIndex::CustomRewind(const interfaces::BlockKey& current_tip, const LOCK(cs_main); const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip.hash)}; const CBlockIndex* new_tip_index{m_chainstate->m_blockman.LookupBlockIndex(new_tip.hash)}; - const auto& consensus_params{Params().GetConsensus()}; do { CBlock block; - if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) { + if (!m_chainstate->m_blockman.ReadBlockFromDisk(block, *iter_tip)) { return error("%s: Failed to read block %s from disk", __func__, iter_tip->GetBlockHash().ToString()); } @@ -409,7 +405,7 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex // Ignore genesis block if (pindex->nHeight > 0) { - if (!UndoReadFromDisk(block_undo, pindex)) { + if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 49bddf2d4d..2e07a35d0d 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -10,8 +10,6 @@ #include <node/blockstorage.h> #include <validation.h> -using node::OpenBlockFile; - constexpr uint8_t DB_TXINDEX{'t'}; std::unique_ptr<TxIndex> g_txindex; @@ -80,7 +78,7 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe return false; } - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + CAutoFile file(m_chainstate->m_blockman.OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); if (file.IsNull()) { return error("%s: OpenBlockFile failed", __func__); } diff --git a/src/init.cpp b/src/init.cpp index 525648b812..38e1dbb4a2 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -18,7 +18,9 @@ #include <blockfilter.h> #include <chain.h> #include <chainparams.h> +#include <chainparamsbase.h> #include <common/args.h> +#include <common/system.h> #include <consensus/amount.h> #include <deploymentstatus.h> #include <hash.h> @@ -44,6 +46,7 @@ #include <node/chainstatemanager_args.h> #include <node/context.h> #include <node/interface_ui.h> +#include <node/kernel_notifications.h> #include <node/mempool_args.h> #include <node/mempool_persist_args.h> #include <node/miner.h> @@ -69,6 +72,7 @@ #include <txdb.h> #include <txmempool.h> #include <util/asmap.h> +#include <util/chaintype.h> #include <util/check.h> #include <util/fs.h> #include <util/fs_helpers.h> @@ -77,7 +81,6 @@ #include <util/string.h> #include <util/syscall_sandbox.h> #include <util/syserror.h> -#include <util/system.h> #include <util/thread.h> #include <util/threadnames.h> #include <util/translation.h> @@ -110,22 +113,25 @@ #include <zmq/zmqrpc.h> #endif +using kernel::DEFAULT_STOPAFTERBLOCKIMPORT; using kernel::DumpMempool; using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; +using node::BlockManager; using node::CacheSizes; using node::CalculateCacheSizes; using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; -using node::DEFAULT_STOPAFTERBLOCKIMPORT; +using node::fReindex; +using node::g_indexes_ready_to_sync; +using node::KernelNotifications; using node::LoadChainstate; using node::MempoolPath; -using node::ShouldPersistMempool; using node::NodeContext; +using node::ShouldPersistMempool; using node::ThreadImport; using node::VerifyLoadedChainstate; -using node::fReindex; static constexpr bool DEFAULT_PROXYRANDOMIZE{true}; static constexpr bool DEFAULT_REST_ENABLE{false}; @@ -326,9 +332,8 @@ void Shutdown(NodeContext& node) #if ENABLE_ZMQ if (g_zmq_notification_interface) { - UnregisterValidationInterface(g_zmq_notification_interface); - delete g_zmq_notification_interface; - g_zmq_notification_interface = nullptr; + UnregisterValidationInterface(g_zmq_notification_interface.get()); + g_zmq_notification_interface.reset(); } #endif @@ -408,14 +413,14 @@ void SetupServerArgs(ArgsManager& argsman) init::AddLoggingArgs(argsman); - const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); - const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); - const auto signetBaseParams = CreateBaseChainParams(CBaseChainParams::SIGNET); - const auto regtestBaseParams = CreateBaseChainParams(CBaseChainParams::REGTEST); - const auto defaultChainParams = CreateChainParams(argsman, CBaseChainParams::MAIN); - const auto testnetChainParams = CreateChainParams(argsman, CBaseChainParams::TESTNET); - const auto signetChainParams = CreateChainParams(argsman, CBaseChainParams::SIGNET); - const auto regtestChainParams = CreateChainParams(argsman, CBaseChainParams::REGTEST); + const auto defaultBaseParams = CreateBaseChainParams(ChainType::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(ChainType::TESTNET); + const auto signetBaseParams = CreateBaseChainParams(ChainType::SIGNET); + const auto regtestBaseParams = CreateBaseChainParams(ChainType::REGTEST); + const auto defaultChainParams = CreateChainParams(argsman, ChainType::MAIN); + const auto testnetChainParams = CreateChainParams(argsman, ChainType::TESTNET); + const auto signetChainParams = CreateChainParams(argsman, ChainType::SIGNET); + const auto regtestChainParams = CreateChainParams(argsman, ChainType::REGTEST); // Hidden Options std::vector<std::string> hidden_args = { @@ -441,6 +446,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE_MB), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -454,7 +460,7 @@ void SetupServerArgs(ArgsManager& argsman) "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk. This will also rebuild active optional indexes.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead. Deactivate all optional indexes before running this.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-settings=<file>", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #if HAVE_SYSTEM argsman.AddArg("-startupnotify=<cmd>", "Execute command on startup.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -794,7 +800,7 @@ std::set<BlockFilterType> g_enabled_filter_types; std::terminate(); }; -bool AppInitBasicSetup(const ArgsManager& args) +bool AppInitBasicSetup(const ArgsManager& args, std::atomic<int>& exit_status) { // ********************************************************* Step 1: setup #ifdef _MSC_VER @@ -808,7 +814,7 @@ bool AppInitBasicSetup(const ArgsManager& args) // Enable heap terminate-on-corruption HeapSetInformation(nullptr, HeapEnableTerminationOnCorruption, nullptr, 0); #endif - if (!InitShutdownState()) { + if (!InitShutdownState(exit_status)) { return InitError(Untranslated("Initializing wait-for-shutdown state failed.")); } @@ -844,14 +850,14 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb // Error if network-specific options (-addnode, -connect, etc) are // specified in default section of config file, but not overridden - // on the command line or in this network's section of the config file. - std::string network = args.GetChainName(); - if (network == CBaseChainParams::SIGNET) { + // on the command line or in this chain's section of the config file. + ChainType chain = args.GetChainType(); + if (chain == ChainType::SIGNET) { LogPrintf("Signet derived magic (message start): %s\n", HexStr(chainparams.MessageStart())); } bilingual_str errors; for (const auto& arg : args.GetUnsuitableSectionOnlyArgs()) { - errors += strprintf(_("Config setting for %s only applied on %s network when in [%s] section.") + Untranslated("\n"), arg, network, network); + errors += strprintf(_("Config setting for %s only applied on %s network when in [%s] section.") + Untranslated("\n"), arg, ChainTypeToString(chain), ChainTypeToString(chain)); } if (!errors.empty()) { @@ -980,19 +986,6 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) return InitError(Untranslated("Unknown rpcserialversion requested.")); - if (args.GetBoolArg("-reindex-chainstate", false)) { - // indexes that must be deactivated to prevent index corruption, see #24630 - if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - return InitError(_("-reindex-chainstate option is not compatible with -coinstatsindex. Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - if (g_enabled_filter_types.count(BlockFilterType::BASIC)) { - return InitError(_("-reindex-chainstate option is not compatible with -blockfilterindex. Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - return InitError(_("-reindex-chainstate option is not compatible with -txindex. Please temporarily disable txindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - } - #if defined(USE_SYSCALL_SANDBOX) if (args.IsArgSet("-sandbox") && !args.IsArgNegated("-sandbox")) { const std::string sandbox_arg{args.GetArg("-sandbox", "")}; @@ -1029,16 +1022,23 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb // Also report errors from parsing before daemonization { + KernelNotifications notifications{}; ChainstateManager::Options chainman_opts_dummy{ .chainparams = chainparams, .datadir = args.GetDataDirNet(), + .notifications = notifications, }; - if (const auto error{ApplyArgsManOptions(args, chainman_opts_dummy)}) { - return InitError(*error); + auto chainman_result{ApplyArgsManOptions(args, chainman_opts_dummy)}; + if (!chainman_result) { + return InitError(util::ErrorString(chainman_result)); } - node::BlockManager::Options blockman_opts_dummy{}; - if (const auto error{ApplyArgsManOptions(args, blockman_opts_dummy)}) { - return InitError(*error); + BlockManager::Options blockman_opts_dummy{ + .chainparams = chainman_opts_dummy.chainparams, + .blocks_dir = args.GetBlocksDirPath(), + }; + auto blockman_result{ApplyArgsManOptions(args, blockman_opts_dummy)}; + if (!blockman_result) { + return InitError(util::ErrorString(blockman_result)); } } @@ -1061,8 +1061,9 @@ static bool LockDataDirectory(bool probeOnly) bool AppInitSanityChecks(const kernel::Context& kernel) { // ********************************************************* Step 4: sanity checks - if (auto error = kernel::SanityChecks(kernel)) { - InitError(*error); + auto result{kernel::SanityChecks(kernel)}; + if (!result) { + InitError(util::ErrorString(result)); return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); } @@ -1237,9 +1238,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Initialize addrman assert(!node.addrman); uiInterface.InitMessage(_("Loading P2P addresses…").translated); - if (const auto error{LoadAddrman(*node.netgroupman, args, node.addrman)}) { - return InitError(*error); - } + auto addrman{LoadAddrman(*node.netgroupman, args)}; + if (!addrman) return InitError(util::ErrorString(addrman)); + node.addrman = std::move(*addrman); } assert(!node.banman); @@ -1358,12 +1359,12 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // -noproxy (or -proxy=0) as well as the empty string can be used to not set a proxy, this is the default std::string proxyArg = args.GetArg("-proxy", ""); if (proxyArg != "" && proxyArg != "0") { - CService proxyAddr; - if (!Lookup(proxyArg, proxyAddr, 9050, fNameLookup)) { + const std::optional<CService> proxyAddr{Lookup(proxyArg, 9050, fNameLookup)}; + if (!proxyAddr.has_value()) { return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); } - Proxy addrProxy = Proxy(proxyAddr, proxyRandomize); + Proxy addrProxy = Proxy(proxyAddr.value(), proxyRandomize); if (!addrProxy.IsValid()) return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); @@ -1389,11 +1390,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) "reaching the Tor network is explicitly forbidden: -onion=0")); } } else { - CService addr; - if (!Lookup(onionArg, addr, 9050, fNameLookup) || !addr.IsValid()) { + const std::optional<CService> addr{Lookup(onionArg, 9050, fNameLookup)}; + if (!addr.has_value() || !addr->IsValid()) { return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); } - onion_proxy = Proxy{addr, proxyRandomize}; + onion_proxy = Proxy{addr.value(), proxyRandomize}; } } @@ -1413,34 +1414,43 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } for (const std::string& strAddr : args.GetArgs("-externalip")) { - CService addrLocal; - if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) - AddLocal(addrLocal, LOCAL_MANUAL); + const std::optional<CService> addrLocal{Lookup(strAddr, GetListenPort(), fNameLookup)}; + if (addrLocal.has_value() && addrLocal->IsValid()) + AddLocal(addrLocal.value(), LOCAL_MANUAL); else return InitError(ResolveErrMsg("externalip", strAddr)); } #if ENABLE_ZMQ - g_zmq_notification_interface = CZMQNotificationInterface::Create(); + g_zmq_notification_interface = CZMQNotificationInterface::Create( + [&chainman = node.chainman](CBlock& block, const CBlockIndex& index) { + assert(chainman); + return chainman->m_blockman.ReadBlockFromDisk(block, index); + }); if (g_zmq_notification_interface) { - RegisterValidationInterface(g_zmq_notification_interface); + RegisterValidationInterface(g_zmq_notification_interface.get()); } #endif // ********************************************************* Step 7: load block chain + node.notifications = std::make_unique<KernelNotifications>(); fReindex = args.GetBoolArg("-reindex", false); bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = args.GetDataDirNet(), .adjusted_time_callback = GetAdjustedTime, + .notifications = *node.notifications, }; - Assert(!ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction + Assert(ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction - node::BlockManager::Options blockman_opts{}; - Assert(!ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction + BlockManager::Options blockman_opts{ + .chainparams = chainman_opts.chainparams, + .blocks_dir = args.GetBlocksDirPath(), + }; + Assert(ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction // cache size calculations CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size()); @@ -1463,8 +1473,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) .estimator = node.fee_estimator.get(), .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, }; - if (const auto err{ApplyArgsManOptions(args, chainparams, mempool_opts)}) { - return InitError(*err); + auto result{ApplyArgsManOptions(args, chainparams, mempool_opts)}; + if (!result) { + return InitError(util::ErrorString(result)); } mempool_opts.check_ratio = std::clamp<int>(mempool_opts.check_ratio, 0, 1'000'000); @@ -1559,9 +1570,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) RegisterValidationInterface(node.peerman.get()); // ********************************************************* Step 8: start indexers + + // If reindex-chainstate was specified, delay syncing indexes until ThreadImport has reindexed the chain + if (!fReindexChainState) g_indexes_ready_to_sync = true; if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - if (const auto error{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}) { - return InitError(*error); + auto result{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}; + if (!result) { + return InitError(util::ErrorString(result)); } g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, fReindex); @@ -1668,7 +1683,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } chainman.m_load_block = std::thread(&util::TraceThread, "loadblk", [=, &chainman, &args] { - ThreadImport(chainman, vImportFiles, args, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); + ThreadImport(chainman, vImportFiles, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); }); // Wait for genesis block to be processed @@ -1739,13 +1754,14 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) }; for (const std::string& bind_arg : args.GetArgs("-bind")) { - CService bind_addr; + std::optional<CService> bind_addr; const size_t index = bind_arg.rfind('='); if (index == std::string::npos) { - if (Lookup(bind_arg, bind_addr, default_bind_port, /*fAllowLookup=*/false)) { - connOptions.vBinds.push_back(bind_addr); - if (IsBadPort(bind_addr.GetPort())) { - InitWarning(BadPortWarning("-bind", bind_addr.GetPort())); + bind_addr = Lookup(bind_arg, default_bind_port, /*fAllowLookup=*/false); + if (bind_addr.has_value()) { + connOptions.vBinds.push_back(bind_addr.value()); + if (IsBadPort(bind_addr.value().GetPort())) { + InitWarning(BadPortWarning("-bind", bind_addr.value().GetPort())); } continue; } @@ -1753,8 +1769,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const std::string network_type = bind_arg.substr(index + 1); if (network_type == "onion") { const std::string truncated_bind_arg = bind_arg.substr(0, index); - if (Lookup(truncated_bind_arg, bind_addr, BaseParams().OnionServiceTargetPort(), false)) { - connOptions.onion_binds.push_back(bind_addr); + bind_addr = Lookup(truncated_bind_arg, BaseParams().OnionServiceTargetPort(), false); + if (bind_addr.has_value()) { + connOptions.onion_binds.push_back(bind_addr.value()); continue; } } @@ -1832,11 +1849,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const std::string& i2psam_arg = args.GetArg("-i2psam", ""); if (!i2psam_arg.empty()) { - CService addr; - if (!Lookup(i2psam_arg, addr, 7656, fNameLookup) || !addr.IsValid()) { + const std::optional<CService> addr{Lookup(i2psam_arg, 7656, fNameLookup)}; + if (!addr.has_value() || !addr->IsValid()) { return InitError(strprintf(_("Invalid -i2psam address or hostname: '%s'"), i2psam_arg)); } - SetProxy(NET_I2P, Proxy{addr}); + SetProxy(NET_I2P, Proxy{addr.value()}); } else { if (args.IsArgSet("-onlynet") && IsReachable(NET_I2P)) { return InitError( diff --git a/src/init.h b/src/init.h index 8c5b2e77d3..fabb4f66d6 100644 --- a/src/init.h +++ b/src/init.h @@ -38,7 +38,7 @@ void InitParameterInteraction(ArgsManager& args); * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read. */ -bool AppInitBasicSetup(const ArgsManager& args); +bool AppInitBasicSetup(const ArgsManager& args, std::atomic<int>& exit_status); /** * Initialization: parameter interaction. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index a3fa753a98..dd664165d3 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -6,8 +6,8 @@ #define BITCOIN_INTERFACES_CHAIN_H #include <blockfilter.h> +#include <common/settings.h> #include <primitives/transaction.h> // For CTransactionRef -#include <util/settings.h> // For util::SettingsValue #include <functional> #include <memory> @@ -87,6 +87,9 @@ struct BlockInfo { unsigned data_pos = 0; const CBlock* data = nullptr; const CBlockUndo* undo_data = nullptr; + // The maximum time in the chain up to and including this block. + // A timestamp that can only move forward. + unsigned int chain_time_max{0}; BlockInfo(const uint256& hash LIFETIMEBOUND) : hash(hash) {} }; @@ -297,17 +300,17 @@ public: virtual int rpcSerializationFlags() = 0; //! Get settings value. - virtual util::SettingsValue getSetting(const std::string& arg) = 0; + virtual common::SettingsValue getSetting(const std::string& arg) = 0; //! Get list of settings values. - virtual std::vector<util::SettingsValue> getSettingsList(const std::string& arg) = 0; + virtual std::vector<common::SettingsValue> getSettingsList(const std::string& arg) = 0; //! Return <datadir>/settings.json setting value. - virtual util::SettingsValue getRwSetting(const std::string& name) = 0; + virtual common::SettingsValue getRwSetting(const std::string& name) = 0; //! Write a setting to <datadir>/settings.json. Optionally just update the //! setting in memory and do not write the file. - virtual bool updateRwSetting(const std::string& name, const util::SettingsValue& value, bool write=true) = 0; + virtual bool updateRwSetting(const std::string& name, const common::SettingsValue& value, bool write=true) = 0; //! Synchronously send transactionAddedToMempool notifications about all //! current mempool transactions to the specified handler and return after diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 7e87d5a523..3f8df57124 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -5,13 +5,13 @@ #ifndef BITCOIN_INTERFACES_NODE_H #define BITCOIN_INTERFACES_NODE_H +#include <common/settings.h> #include <consensus/amount.h> // For CAmount #include <net.h> // For NodeId #include <net_types.h> // For banmap_t #include <netaddress.h> // For Network #include <netbase.h> // For ConnectionDirection #include <support/allocators/secure.h> // For SecureString -#include <util/settings.h> // For util::SettingsValue #include <util/translation.h> #include <functional> @@ -80,6 +80,9 @@ public: //! Get warnings. virtual bilingual_str getWarnings() = 0; + //! Get exit status. + virtual int getExitStatus() = 0; + // Get log flags. virtual uint32_t getLogCategories() = 0; @@ -103,14 +106,14 @@ public: virtual bool isSettingIgnored(const std::string& name) = 0; //! Return setting value from <datadir>/settings.json or bitcoin.conf. - virtual util::SettingsValue getPersistentSetting(const std::string& name) = 0; + virtual common::SettingsValue getPersistentSetting(const std::string& name) = 0; //! Update a setting in <datadir>/settings.json. - virtual void updateRwSetting(const std::string& name, const util::SettingsValue& value) = 0; + virtual void updateRwSetting(const std::string& name, const common::SettingsValue& value) = 0; //! Force a setting value to be applied, overriding any other configuration //! source, but not being persisted. - virtual void forceSetting(const std::string& name, const util::SettingsValue& value) = 0; + virtual void forceSetting(const std::string& name, const common::SettingsValue& value) = 0; //! Clear all settings in <datadir>/settings.json and store a backup of //! previous settings in <datadir>/settings.json.bak. diff --git a/src/ipc/interfaces.cpp b/src/ipc/interfaces.cpp index d804d9d291..e446cc98db 100644 --- a/src/ipc/interfaces.cpp +++ b/src/ipc/interfaces.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <interfaces/init.h> #include <interfaces/ipc.h> #include <ipc/capnp/protocol.h> @@ -10,7 +11,6 @@ #include <logging.h> #include <tinyformat.h> #include <util/fs.h> -#include <util/system.h> #include <cstdio> #include <cstdlib> diff --git a/src/kernel/blockmanager_opts.h b/src/kernel/blockmanager_opts.h index 9dc93b6dd2..8f26422f72 100644 --- a/src/kernel/blockmanager_opts.h +++ b/src/kernel/blockmanager_opts.h @@ -5,14 +5,26 @@ #ifndef BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H #define BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H +#include <util/fs.h> + +#include <cstdint> + +class CChainParams; + namespace kernel { +static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; + /** * An options struct for `BlockManager`, more ergonomically referred to as * `BlockManager::Options` due to the using-declaration in `BlockManager`. */ struct BlockManagerOpts { + const CChainParams& chainparams; uint64_t prune_target{0}; + bool fast_prune{false}; + bool stop_after_block_import{DEFAULT_STOPAFTERBLOCKIMPORT}; + const fs::path blocks_dir; }; } // namespace kernel diff --git a/src/kernel/chain.cpp b/src/kernel/chain.cpp index 82e77125d7..1c877866d0 100644 --- a/src/kernel/chain.cpp +++ b/src/kernel/chain.cpp @@ -16,6 +16,7 @@ interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* index, const CBlock* data if (index) { info.prev_hash = index->pprev ? index->pprev->phashBlock : nullptr; info.height = index->nHeight; + info.chain_time_max = index->GetBlockTimeMax(); LOCK(::cs_main); info.file_number = index->nFile; info.data_pos = index->nDataPos; diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 2827737ee1..d9ed1547b3 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -10,13 +10,13 @@ #include <consensus/merkle.h> #include <consensus/params.h> #include <hash.h> -#include <chainparamsbase.h> #include <logging.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <script/interpreter.h> #include <script/script.h> #include <uint256.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <algorithm> @@ -70,7 +70,7 @@ static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits class CMainParams : public CChainParams { public: CMainParams() { - strNetworkID = CBaseChainParams::MAIN; + m_chain_type = ChainType::MAIN; consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 210000; @@ -192,7 +192,7 @@ public: class CTestNetParams : public CChainParams { public: CTestNetParams() { - strNetworkID = CBaseChainParams::TESTNET; + m_chain_type = ChainType::TESTNET; consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 210000; @@ -328,7 +328,7 @@ public: vSeeds = *options.seeds; } - strNetworkID = CBaseChainParams::SIGNET; + m_chain_type = ChainType::SIGNET; consensus.signet_blocks = true; consensus.signet_challenge.assign(bin.begin(), bin.end()); consensus.nSubsidyHalvingInterval = 210000; @@ -397,7 +397,7 @@ class CRegTestParams : public CChainParams public: explicit CRegTestParams(const RegTestOptions& opts) { - strNetworkID = CBaseChainParams::REGTEST; + m_chain_type = ChainType::REGTEST; consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 150; diff --git a/src/kernel/chainparams.h b/src/kernel/chainparams.h index 32fe618dbd..ad0b49a885 100644 --- a/src/kernel/chainparams.h +++ b/src/kernel/chainparams.h @@ -11,6 +11,7 @@ #include <primitives/block.h> #include <protocol.h> #include <uint256.h> +#include <util/chaintype.h> #include <util/hash_type.h> #include <cstdint> @@ -114,8 +115,10 @@ public: uint64_t AssumedChainStateSize() const { return m_assumed_chain_state_size; } /** Whether it is possible to mine blocks on demand (no retargeting) */ bool MineBlocksOnDemand() const { return consensus.fPowNoRetargeting; } - /** Return the network string */ - std::string NetworkIDString() const { return strNetworkID; } + /** Return the chain type string */ + std::string GetChainTypeString() const { return ChainTypeToString(m_chain_type); } + /** Return the chain type */ + ChainType GetChainType() const { return m_chain_type; } /** Return the list of hostnames to look up for DNS seeds */ const std::vector<std::string>& DNSSeeds() const { return vSeeds; } const std::vector<unsigned char>& Base58Prefix(Base58Type type) const { return base58Prefixes[type]; } @@ -172,7 +175,7 @@ protected: std::vector<std::string> vSeeds; std::vector<unsigned char> base58Prefixes[MAX_BASE58_TYPES]; std::string bech32_hrp; - std::string strNetworkID; + ChainType m_chain_type; CBlock genesis; std::vector<uint8_t> vFixedSeeds; bool fDefaultConsistencyChecks; diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 2395f60164..035a913d10 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H #define BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H +#include <kernel/notifications_interface.h> + #include <arith_uint256.h> #include <dbwrapper.h> #include <txdb.h> @@ -19,6 +21,7 @@ class CChainParams; static constexpr bool DEFAULT_CHECKPOINTS_ENABLED{true}; static constexpr auto DEFAULT_MAX_TIP_AGE{24h}; +static constexpr int DEFAULT_STOPATHEIGHT{0}; namespace kernel { @@ -42,6 +45,8 @@ struct ChainstateManagerOpts { DBOptions block_tree_db{}; DBOptions coins_db{}; CoinsViewOptions coins_view{}; + Notifications& notifications; + int stop_at_height{DEFAULT_STOPATHEIGHT}; }; } // namespace kernel diff --git a/src/kernel/checks.cpp b/src/kernel/checks.cpp index 4c303c172c..bf8a2ec74c 100644 --- a/src/kernel/checks.cpp +++ b/src/kernel/checks.cpp @@ -13,21 +13,21 @@ namespace kernel { -std::optional<bilingual_str> SanityChecks(const Context&) +util::Result<void> SanityChecks(const Context&) { if (!ECC_InitSanityCheck()) { - return Untranslated("Elliptic curve cryptography sanity check failure. Aborting."); + return util::Error{Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")}; } if (!Random_SanityCheck()) { - return Untranslated("OS cryptographic RNG sanity check failure. Aborting."); + return util::Error{Untranslated("OS cryptographic RNG sanity check failure. Aborting.")}; } if (!ChronoSanityCheck()) { - return Untranslated("Clock epoch mismatch. Aborting."); + return util::Error{Untranslated("Clock epoch mismatch. Aborting.")}; } - return std::nullopt; + return {}; } } diff --git a/src/kernel/checks.h b/src/kernel/checks.h index 3eb14824fb..fd8c167015 100644 --- a/src/kernel/checks.h +++ b/src/kernel/checks.h @@ -5,9 +5,7 @@ #ifndef BITCOIN_KERNEL_CHECKS_H #define BITCOIN_KERNEL_CHECKS_H -#include <optional> - -struct bilingual_str; +#include <util/result.h> namespace kernel { @@ -16,8 +14,7 @@ struct Context; /** * Ensure a usable environment with all necessary library support. */ -std::optional<bilingual_str> SanityChecks(const Context&); - -} +[[nodiscard]] util::Result<void> SanityChecks(const Context&); +} // namespace kernel #endif // BITCOIN_KERNEL_CHECKS_H diff --git a/src/kernel/coinstats.cpp b/src/kernel/coinstats.cpp index 4b75c387a6..527433f45e 100644 --- a/src/kernel/coinstats.cpp +++ b/src/kernel/coinstats.cpp @@ -123,7 +123,7 @@ static bool ComputeUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, c uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { - interruption_point(); + if (interruption_point) interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { diff --git a/src/kernel/mempool_entry.h b/src/kernel/mempool_entry.h index e1ba4296ef..969ddcd1ce 100644 --- a/src/kernel/mempool_entry.h +++ b/src/kernel/mempool_entry.h @@ -75,7 +75,7 @@ private: mutable Parents m_parents; mutable Children m_children; const CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups - const size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) + const int32_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) const size_t nUsageSize; //!< ... and total memory usage const int64_t nTime; //!< Local time when entering the mempool const unsigned int entryHeight; //!< Chain height when entering the mempool @@ -88,12 +88,14 @@ private: // mempool; if we remove this transaction we must remove all of these // descendants as well. uint64_t nCountWithDescendants{1}; //!< number of descendant transactions - uint64_t nSizeWithDescendants; //!< ... and size + // Using int64_t instead of int32_t to avoid signed integer overflow issues. + int64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) // Analogous statistics for ancestor transactions uint64_t nCountWithAncestors{1}; - uint64_t nSizeWithAncestors; + // Using int64_t instead of int32_t to avoid signed integer overflow issues. + int64_t nSizeWithAncestors; CAmount nModFeesWithAncestors; int64_t nSigOpCostWithAncestors; @@ -104,7 +106,7 @@ public: int64_t sigops_cost, LockPoints lp) : tx{tx}, nFee{fee}, - nTxWeight(GetTransactionWeight(*tx)), + nTxWeight{GetTransactionWeight(*tx)}, nUsageSize{RecursiveDynamicUsage(tx)}, nTime{time}, entryHeight{entry_height}, @@ -121,11 +123,11 @@ public: const CTransaction& GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } const CAmount& GetFee() const { return nFee; } - size_t GetTxSize() const + int32_t GetTxSize() const { return GetVirtualTransactionSize(nTxWeight, sigOpCost, ::nBytesPerSigOp); } - size_t GetTxWeight() const { return nTxWeight; } + int32_t GetTxWeight() const { return nTxWeight; } std::chrono::seconds GetTime() const { return std::chrono::seconds{nTime}; } unsigned int GetHeight() const { return entryHeight; } int64_t GetSigOpCost() const { return sigOpCost; } @@ -134,9 +136,9 @@ public: const LockPoints& GetLockPoints() const { return lockPoints; } // Adjusts the descendant state. - void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); + void UpdateDescendantState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state - void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); + void UpdateAncestorState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); // Updates the modified fees with descendants/ancestors. void UpdateModifiedFee(CAmount fee_diff) { @@ -152,13 +154,13 @@ public: } uint64_t GetCountWithDescendants() const { return nCountWithDescendants; } - uint64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } + int64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } CAmount GetModFeesWithDescendants() const { return nModFeesWithDescendants; } bool GetSpendsCoinbase() const { return spendsCoinbase; } uint64_t GetCountWithAncestors() const { return nCountWithAncestors; } - uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } + int64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } int64_t GetSigOpCostWithAncestors() const { return nSigOpCostWithAncestors; } diff --git a/src/kernel/notifications_interface.h b/src/kernel/notifications_interface.h new file mode 100644 index 0000000000..48248e9aa0 --- /dev/null +++ b/src/kernel/notifications_interface.h @@ -0,0 +1,33 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H +#define BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H + +#include <cstdint> +#include <string> + +class CBlockIndex; +enum class SynchronizationState; +struct bilingual_str; + +namespace kernel { + +/** + * A base class defining functions for notifying about certain kernel + * events. + */ +class Notifications +{ +public: + virtual ~Notifications(){}; + + virtual void blockTip(SynchronizationState state, CBlockIndex& index) {} + virtual void headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) {} + virtual void progress(const bilingual_str& title, int progress_percent, bool resume_possible) {} + virtual void warning(const bilingual_str& warning) {} +}; +} // namespace kernel + +#endif // BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H diff --git a/src/key_io.cpp b/src/key_io.cpp index 4659a59544..454a96df5e 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -124,7 +124,11 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par data.clear(); const auto dec = bech32::Decode(str); - if ((dec.encoding == bech32::Encoding::BECH32 || dec.encoding == bech32::Encoding::BECH32M) && dec.data.size() > 0) { + if (dec.encoding == bech32::Encoding::BECH32 || dec.encoding == bech32::Encoding::BECH32M) { + if (dec.data.empty()) { + error_str = "Empty Bech32 data section"; + return CNoDestination(); + } // Bech32 decoding if (dec.hrp != params.Bech32HRP()) { error_str = strprintf("Invalid or unsupported prefix for Segwit (Bech32) address (expected %s, got %s).", params.Bech32HRP(), dec.hrp); @@ -142,6 +146,9 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par // The rest of the symbols are converted witness program bytes. data.reserve(((dec.data.size() - 1) * 5) / 8); if (ConvertBits<5, 8, false>([&](unsigned char c) { data.push_back(c); }, dec.data.begin() + 1, dec.data.end())) { + + std::string_view byte_str{data.size() == 1 ? "byte" : "bytes"}; + if (version == 0) { { WitnessV0KeyHash keyid; @@ -158,7 +165,7 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par } } - error_str = "Invalid Bech32 v0 address data size"; + error_str = strprintf("Invalid Bech32 v0 address program size (%d %s), per BIP141", data.size(), byte_str); return CNoDestination(); } @@ -175,7 +182,7 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par } if (data.size() < 2 || data.size() > BECH32_WITNESS_PROG_MAX_LEN) { - error_str = "Invalid Bech32 address data size"; + error_str = strprintf("Invalid Bech32 address program size (%d %s)", data.size(), byte_str); return CNoDestination(); } @@ -184,6 +191,9 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par std::copy(data.begin(), data.end(), unk.program); unk.length = data.size(); return unk; + } else { + error_str = strprintf("Invalid padding in Bech32 data section"); + return CNoDestination(); } } diff --git a/src/mapport.cpp b/src/mapport.cpp index 994fd12cf5..118827901a 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -9,12 +9,12 @@ #include <mapport.h> #include <clientversion.h> +#include <common/system.h> #include <logging.h> #include <net.h> #include <netaddress.h> #include <netbase.h> #include <util/syscall_sandbox.h> -#include <util/system.h> #include <util/thread.h> #include <util/threadinterrupt.h> @@ -175,10 +175,10 @@ static bool ProcessUpnp() LogPrintf("UPnP: GetExternalIPAddress() returned %d\n", r); } else { if (externalIPAddress[0]) { - CNetAddr resolved; - if (LookupHost(externalIPAddress, resolved, false)) { - LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToStringAddr()); - AddLocal(resolved, LOCAL_MAPPED); + std::optional<CNetAddr> resolved{LookupHost(externalIPAddress, false)}; + if (resolved.has_value()) { + LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved->ToStringAddr()); + AddLocal(resolved.value(), LOCAL_MAPPED); } } else { LogPrintf("UPnP: GetExternalIPAddress failed.\n"); diff --git a/src/net.cpp b/src/net.cpp index 337cf60680..282c8fb741 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -130,14 +130,10 @@ uint16_t GetListenPort() { // If -bind= is provided with ":port" part, use that (first one if multiple are provided). for (const std::string& bind_arg : gArgs.GetArgs("-bind")) { - CService bind_addr; constexpr uint16_t dummy_port = 0; - if (Lookup(bind_arg, bind_addr, dummy_port, /*fAllowLookup=*/false)) { - if (bind_addr.GetPort() != dummy_port) { - return bind_addr.GetPort(); - } - } + const std::optional<CService> bind_addr{Lookup(bind_arg, dummy_port, /*fAllowLookup=*/false)}; + if (bind_addr.has_value() && bind_addr->GetPort() != dummy_port) return bind_addr->GetPort(); } // Otherwise, if -whitebind= without NetPermissionFlags::NoBan is provided, use that @@ -461,9 +457,9 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo const uint16_t default_port{pszDest != nullptr ? Params().GetDefaultPort(pszDest) : Params().GetDefaultPort()}; if (pszDest) { - std::vector<CService> resolved; - if (Lookup(pszDest, resolved, default_port, fNameLookup && !HaveNameProxy(), 256) && !resolved.empty()) { - const CService rnd{resolved[GetRand(resolved.size())]}; + const std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; + if (!resolved.empty()) { + const CService& rnd{resolved[GetRand(resolved.size())]}; addrConnect = CAddress{MaybeFlipIPv6toCJDNS(rnd), NODE_NONE}; if (!addrConnect.IsValid()) { LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); @@ -1487,7 +1483,6 @@ void CConnman::ThreadDNSAddressSeed() if (HaveNameProxy()) { AddAddrFetch(seed); } else { - std::vector<CNetAddr> vIPs; std::vector<CAddress> vAdd; ServiceFlags requiredServiceBits = GetDesirableServiceFlags(NODE_NONE); std::string host = strprintf("x%x.%s", requiredServiceBits, seed); @@ -1496,8 +1491,9 @@ void CConnman::ThreadDNSAddressSeed() continue; } unsigned int nMaxIPs = 256; // Limits number of IPs learned from a DNS seed - if (LookupHost(host, vIPs, nMaxIPs, true)) { - for (const CNetAddr& ip : vIPs) { + const auto addresses{LookupHost(host, nMaxIPs, true)}; + if (!addresses.empty()) { + for (const CNetAddr& ip : addresses) { CAddress addr = CAddress(CService(ip, Params().GetDefaultPort()), requiredServiceBits); addr.nTime = rng.rand_uniform_delay(Now<NodeSeconds>() - 3 * 24h, -4 * 24h); // use a random age between 3 and 7 days old vAdd.push_back(addr); @@ -1707,7 +1703,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) int nOutboundFullRelay = 0; int nOutboundBlockRelay = 0; int outbound_privacy_network_peers = 0; - std::set<std::vector<unsigned char>> setConnected; // netgroups of our ipv4/ipv6 outbound peers + std::set<std::vector<unsigned char>> outbound_ipv46_peer_netgroups; { LOCK(m_nodes_mutex); @@ -1729,7 +1725,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) case ConnectionType::MANUAL: case ConnectionType::OUTBOUND_FULL_RELAY: case ConnectionType::BLOCK_RELAY: - CAddress address{pnode->addr}; + const CAddress address{pnode->addr}; if (address.IsTor() || address.IsI2P() || address.IsCJDNS()) { // Since our addrman-groups for these networks are // random, without relation to the route we @@ -1740,7 +1736,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // these networks. ++outbound_privacy_network_peers; } else { - setConnected.insert(m_netgroupman.GetGroup(address)); + outbound_ipv46_peer_netgroups.insert(m_netgroupman.GetGroup(address)); } } // no default case, so the compiler can warn about missing cases } @@ -1815,7 +1811,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) m_anchors.pop_back(); if (!addr.IsValid() || IsLocal(addr) || !IsReachable(addr) || !HasAllDesirableServiceFlags(addr.nServices) || - setConnected.count(m_netgroupman.GetGroup(addr))) continue; + outbound_ipv46_peer_netgroups.count(m_netgroupman.GetGroup(addr))) continue; addrConnect = addr; LogPrint(BCLog::NET, "Trying to make an anchor connection to %s\n", addrConnect.ToStringAddrPort()); break; @@ -1855,8 +1851,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) std::tie(addr, addr_last_try) = addrman.Select(); } - // Require outbound connections, other than feelers, to be to distinct network groups - if (!fFeeler && setConnected.count(m_netgroupman.GetGroup(addr))) { + // Require outbound IPv4/IPv6 connections, other than feelers, to be to distinct network groups + if (!fFeeler && outbound_ipv46_peer_netgroups.count(m_netgroupman.GetGroup(addr))) { break; } @@ -1902,8 +1898,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Record addrman failure attempts when node has at least 2 persistent outbound connections to peers with // different netgroups in ipv4/ipv6 networks + all peers in Tor/I2P/CJDNS networks. // Don't record addrman failure attempts when node is offline. This can be identified since all local - // network connections(if any) belong in the same netgroup and size of setConnected would only be 1. - OpenNetworkConnection(addrConnect, (int)setConnected.size() + outbound_privacy_network_peers >= std::min(nMaxConnections - 1, 2), &grant, nullptr, conn_type); + // network connections (if any) belong in the same netgroup, and the size of `outbound_ipv46_peer_netgroups` would only be 1. + const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(nMaxConnections - 1, 2)}; + OpenNetworkConnection(addrConnect, count_failures, &grant, /*strDest=*/nullptr, conn_type); } } } @@ -2201,14 +2198,11 @@ void Discover() char pszHostName[256] = ""; if (gethostname(pszHostName, sizeof(pszHostName)) != SOCKET_ERROR) { - std::vector<CNetAddr> vaddr; - if (LookupHost(pszHostName, vaddr, 0, true)) + const std::vector<CNetAddr> addresses{LookupHost(pszHostName, 0, true)}; + for (const CNetAddr& addr : addresses) { - for (const CNetAddr &addr : vaddr) - { - if (AddLocal(addr, LOCAL_IF)) - LogPrintf("%s: %s - %s\n", __func__, pszHostName, addr.ToStringAddr()); - } + if (AddLocal(addr, LOCAL_IF)) + LogPrintf("%s: %s - %s\n", __func__, pszHostName, addr.ToStringAddr()); } } #elif (HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS) @@ -200,7 +200,9 @@ public: int nVersion; std::string cleanSubVer; bool fInbound; + // We requested high bandwidth connection to peer bool m_bip152_highbandwidth_to; + // Peer requested high bandwidth connection bool m_bip152_highbandwidth_from; int m_starting_height; uint64_t nSendBytes; diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index f829e56aa2..23226bbb4f 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <net_permissions.h> #include <netbase.h> #include <util/error.h> -#include <util/system.h> #include <util/translation.h> const std::vector<std::string> NET_PERMISSIONS_DOC{ @@ -88,18 +88,18 @@ bool NetWhitebindPermissions::TryParse(const std::string& str, NetWhitebindPermi if (!TryParsePermissionFlags(str, flags, offset, error)) return false; const std::string strBind = str.substr(offset); - CService addrBind; - if (!Lookup(strBind, addrBind, 0, false)) { + const std::optional<CService> addrBind{Lookup(strBind, 0, false)}; + if (!addrBind.has_value()) { error = ResolveErrMsg("whitebind", strBind); return false; } - if (addrBind.GetPort() == 0) { + if (addrBind.value().GetPort() == 0) { error = strprintf(_("Need to specify a port with -whitebind: '%s'"), strBind); return false; } output.m_flags = flags; - output.m_service = addrBind; + output.m_service = addrBind.value(); error = Untranslated(""); return true; } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index c50aa2e4f9..6597019797 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -51,12 +51,7 @@ #include <optional> #include <typeinfo> -using node::ReadBlockFromDisk; -using node::ReadRawBlockFromDisk; - -/** How long to cache transactions in mapRelay for normal relay */ -static constexpr auto RELAY_TX_CACHE_TIME = 15min; -/** How long a transaction has to be in the mempool before it can unconditionally be relayed (even when not in mapRelay). */ +/** How long a transaction has to be in the mempool before it can unconditionally be relayed. */ static constexpr auto UNCONDITIONAL_RELAY_DELAY = 2min; /** Headers download timeout. * Timeout = base + per_header * (expected number of headers) */ @@ -292,7 +287,8 @@ struct Peer { * this does not have to be sorted. */ std::set<uint256> m_tx_inventory_to_send GUARDED_BY(m_tx_inventory_mutex); /** Whether the peer has requested us to send our complete mempool. Only - * permitted if the peer has NetPermissionFlags::Mempool. See BIP35. */ + * permitted if the peer has NetPermissionFlags::Mempool or we advertise + * NODE_BLOOM. See BIP35. */ bool m_send_mempool GUARDED_BY(m_tx_inventory_mutex){false}; /** The last time a BIP35 `mempool` request was serviced. */ std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; @@ -433,7 +429,6 @@ struct CNodeState { std::list<QueuedBlock> vBlocksInFlight; //! When the first entry in vBlocksInFlight started downloading. Don't care when vBlocksInFlight is empty. std::chrono::microseconds m_downloading_since{0us}; - int nBlocksInFlight{0}; //! Whether we consider this a preferred download peer. bool fPreferredDownload{false}; /** Whether this peer wants invs or cmpctblocks (when possible) for block announcements. */ @@ -854,6 +849,7 @@ private: std::shared_ptr<const CBlock> m_most_recent_block GUARDED_BY(m_most_recent_block_mutex); std::shared_ptr<const CBlockHeaderAndShortTxIDs> m_most_recent_compact_block GUARDED_BY(m_most_recent_block_mutex); uint256 m_most_recent_block_hash GUARDED_BY(m_most_recent_block_mutex); + std::unique_ptr<const std::map<uint256, CTransactionRef>> m_most_recent_block_txs GUARDED_BY(m_most_recent_block_mutex); // Data about the low-work headers synchronization, aggregated from all peers' HeadersSyncStates. /** Mutex guarding the other m_headers_presync_* variables. */ @@ -879,11 +875,17 @@ private: /** Have we requested this block from a peer */ bool IsBlockRequested(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Have we requested this block from an outbound peer */ + bool IsBlockRequestedFromOutbound(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Remove this block from our tracked requested blocks. Called if: * - the block has been received from a peer * - the request for the block has timed out + * If "from_peer" is specified, then only remove the block if it is in + * flight from that peer (to avoid one peer's network traffic from + * affecting another's state). */ - void RemoveBlockRequest(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void RemoveBlockRequest(const uint256& hash, std::optional<NodeId> from_peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /* Mark a block as in flight * Returns false, still setting pit, if the block was already in flight from the same peer @@ -898,14 +900,16 @@ private: */ void FindNextBlocksToDownload(const Peer& peer, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> > mapBlocksInFlight GUARDED_BY(cs_main); + /* Multimap used to preserve insertion order */ + typedef std::multimap<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator>> BlockDownloadMap; + BlockDownloadMap mapBlocksInFlight GUARDED_BY(cs_main); /** When our tip was last updated. */ std::atomic<std::chrono::seconds> m_last_tip_update{0s}; /** Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). */ CTransactionRef FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) - EXCLUSIVE_LOCKS_REQUIRED(NetEventsInterface::g_msgproc_mutex); + EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex, NetEventsInterface::g_msgproc_mutex); void ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex, peer.m_getdata_requests_mutex, NetEventsInterface::g_msgproc_mutex) @@ -914,12 +918,6 @@ private: /** Process a new block. Perform any post-processing housekeeping */ void ProcessBlock(CNode& node, const std::shared_ptr<const CBlock>& block, bool force_processing, bool min_pow_checked); - /** Relay map (txid or wtxid -> CTransactionRef) */ - typedef std::map<uint256, CTransactionRef> MapRelay; - MapRelay mapRelay GUARDED_BY(NetEventsInterface::g_msgproc_mutex); - /** Expiration-time ordered list of (expire time, relay map entry) pairs. */ - std::deque<std::pair<std::chrono::microseconds, MapRelay::iterator>> g_relay_expiration GUARDED_BY(NetEventsInterface::g_msgproc_mutex); - /** * When a peer sends us a valid block, instruct it to announce blocks to us * using CMPCTBLOCK if possible by adding its nodeid to the end of @@ -1116,34 +1114,55 @@ std::chrono::microseconds PeerManagerImpl::NextInvToInbounds(std::chrono::micros bool PeerManagerImpl::IsBlockRequested(const uint256& hash) { - return mapBlocksInFlight.find(hash) != mapBlocksInFlight.end(); + return mapBlocksInFlight.count(hash); } -void PeerManagerImpl::RemoveBlockRequest(const uint256& hash) +bool PeerManagerImpl::IsBlockRequestedFromOutbound(const uint256& hash) { - auto it = mapBlocksInFlight.find(hash); - if (it == mapBlocksInFlight.end()) { - // Block was not requested - return; + for (auto range = mapBlocksInFlight.equal_range(hash); range.first != range.second; range.first++) { + auto [nodeid, block_it] = range.first->second; + CNodeState& nodestate = *Assert(State(nodeid)); + if (!nodestate.m_is_inbound) return true; } - auto [node_id, list_it] = it->second; - CNodeState *state = State(node_id); - assert(state != nullptr); + return false; +} - if (state->vBlocksInFlight.begin() == list_it) { - // First block on the queue was received, update the start download time for the next one - state->m_downloading_since = std::max(state->m_downloading_since, GetTime<std::chrono::microseconds>()); +void PeerManagerImpl::RemoveBlockRequest(const uint256& hash, std::optional<NodeId> from_peer) +{ + auto range = mapBlocksInFlight.equal_range(hash); + if (range.first == range.second) { + // Block was not requested from any peer + return; } - state->vBlocksInFlight.erase(list_it); - state->nBlocksInFlight--; - if (state->nBlocksInFlight == 0) { - // Last validated block on the queue was received. - m_peers_downloading_from--; + // We should not have requested too many of this block + Assume(mapBlocksInFlight.count(hash) <= MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK); + + while (range.first != range.second) { + auto [node_id, list_it] = range.first->second; + + if (from_peer && *from_peer != node_id) { + range.first++; + continue; + } + + CNodeState& state = *Assert(State(node_id)); + + if (state.vBlocksInFlight.begin() == list_it) { + // First block on the queue was received, update the start download time for the next one + state.m_downloading_since = std::max(state.m_downloading_since, GetTime<std::chrono::microseconds>()); + } + state.vBlocksInFlight.erase(list_it); + + if (state.vBlocksInFlight.empty()) { + // Last validated block on the queue for this peer was received. + m_peers_downloading_from--; + } + state.m_stalling_since = 0us; + + range.first = mapBlocksInFlight.erase(range.first); } - state->m_stalling_since = 0us; - mapBlocksInFlight.erase(it); } bool PeerManagerImpl::BlockRequested(NodeId nodeid, const CBlockIndex& block, std::list<QueuedBlock>::iterator** pit) @@ -1153,27 +1172,29 @@ bool PeerManagerImpl::BlockRequested(NodeId nodeid, const CBlockIndex& block, st CNodeState *state = State(nodeid); assert(state != nullptr); + Assume(mapBlocksInFlight.count(hash) <= MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK); + // Short-circuit most stuff in case it is from the same node - std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); - if (itInFlight != mapBlocksInFlight.end() && itInFlight->second.first == nodeid) { - if (pit) { - *pit = &itInFlight->second.second; + for (auto range = mapBlocksInFlight.equal_range(hash); range.first != range.second; range.first++) { + if (range.first->second.first == nodeid) { + if (pit) { + *pit = &range.first->second.second; + } + return false; } - return false; } - // Make sure it's not listed somewhere already. - RemoveBlockRequest(hash); + // Make sure it's not being fetched already from same peer. + RemoveBlockRequest(hash, nodeid); std::list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), {&block, std::unique_ptr<PartiallyDownloadedBlock>(pit ? new PartiallyDownloadedBlock(&m_mempool) : nullptr)}); - state->nBlocksInFlight++; - if (state->nBlocksInFlight == 1) { + if (state->vBlocksInFlight.size() == 1) { // We're starting a block download (batch) from this peer. state->m_downloading_since = GetTime<std::chrono::microseconds>(); m_peers_downloading_from++; } - itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))).first; + auto itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))); if (pit) { *pit = &itInFlight->second.second; } @@ -1376,7 +1397,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int co } } else if (waitingfor == -1) { // This is the first already-in-flight block. - waitingfor = mapBlocksInFlight[pindex->GetBlockHash()].first; + waitingfor = mapBlocksInFlight.lower_bound(pindex->GetBlockHash())->second.first; } } } @@ -1506,13 +1527,21 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) nSyncStarted--; for (const QueuedBlock& entry : state->vBlocksInFlight) { - mapBlocksInFlight.erase(entry.pindex->GetBlockHash()); + auto range = mapBlocksInFlight.equal_range(entry.pindex->GetBlockHash()); + while (range.first != range.second) { + auto [node_id, list_it] = range.first->second; + if (node_id != nodeid) { + range.first++; + } else { + range.first = mapBlocksInFlight.erase(range.first); + } + } } m_orphanage.EraseForPeer(nodeid); m_txrequest.DisconnectedPeer(nodeid); if (m_txreconciliation) m_txreconciliation->ForgetPeer(nodeid); m_num_preferred_download_peers -= state->fPreferredDownload; - m_peers_downloading_from -= (state->nBlocksInFlight != 0); + m_peers_downloading_from -= (!state->vBlocksInFlight.empty()); assert(m_peers_downloading_from >= 0); m_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; assert(m_outbound_peers_with_protect_from_disconnect >= 0); @@ -1753,11 +1782,10 @@ std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBl LOCK(cs_main); - // Mark block as in-flight unless it already is (for this peer). - // If the peer does not send us a block, vBlocksInFlight remains non-empty, - // causing us to timeout and disconnect. - // If a block was already in-flight for a different peer, its BLOCKTXN - // response will be dropped. + // Forget about all prior requests + RemoveBlockRequest(block_index.GetBlockHash(), std::nullopt); + + // Mark block as in-flight if (!BlockRequested(peer_id, block_index)) return "Already requested from this peer"; // Construct message to request the block @@ -1892,10 +1920,17 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha std::async(std::launch::deferred, [&] { return msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock); })}; { + auto most_recent_block_txs = std::make_unique<std::map<uint256, CTransactionRef>>(); + for (const auto& tx : pblock->vtx) { + most_recent_block_txs->emplace(tx->GetHash(), tx); + most_recent_block_txs->emplace(tx->GetWitnessHash(), tx); + } + LOCK(m_most_recent_block_mutex); m_most_recent_block_hash = hashBlock; m_most_recent_block = pblock; m_most_recent_compact_block = pcmpctblock; + m_most_recent_block_txs = std::move(most_recent_block_txs); } m_connman.ForEachNode([this, pindex, &lazy_ser, &hashBlock](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { @@ -2179,7 +2214,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& // Fast-path: in this case it is possible to serve the block directly from disk, // as the network format matches the format on disk std::vector<uint8_t> block_data; - if (!ReadRawBlockFromDisk(block_data, pindex->GetBlockPos(), m_chainparams.MessageStart())) { + if (!m_chainman.m_blockman.ReadRawBlockFromDisk(block_data, pindex->GetBlockPos(), m_chainparams.MessageStart())) { assert(!"cannot load block from disk"); } m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, Span{block_data})); @@ -2187,7 +2222,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } else { // Send block from disk std::shared_ptr<CBlock> pblockRead = std::make_shared<CBlock>(); - if (!ReadBlockFromDisk(*pblockRead, pindex, m_chainparams.GetConsensus())) { + if (!m_chainman.m_blockman.ReadBlockFromDisk(*pblockRead, *pindex)) { assert(!"cannot load block from disk"); } pblock = pblockRead; @@ -2266,13 +2301,17 @@ CTransactionRef PeerManagerImpl::FindTxForGetData(const Peer::TxRelay& tx_relay, } } - // Otherwise, the transaction must have been announced recently. - if (tx_relay.m_recently_announced_invs.contains(gtxid.GetHash())) { - // If it was, it can be relayed from either the mempool... - if (txinfo.tx) return std::move(txinfo.tx); - // ... or the relay pool. - auto mi = mapRelay.find(gtxid.GetHash()); - if (mi != mapRelay.end()) return mi->second; + // Otherwise, the transaction might have been announced recently. + bool recent = tx_relay.m_recently_announced_invs.contains(gtxid.GetHash()); + if (recent && txinfo.tx) return std::move(txinfo.tx); + + // Or it might be from the most recent block + { + LOCK(m_most_recent_block_mutex); + if (m_most_recent_block_txs != nullptr) { + auto it = m_most_recent_block_txs->find(gtxid.GetHash()); + if (it != m_most_recent_block_txs->end()) return it->second; + } } return {}; @@ -2673,7 +2712,7 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, c std::vector<CInv> vGetData; // Download as much as possible, from earliest to latest. for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (nodestate->vBlocksInFlight.size() >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { // Can't download any more from this peer break; } @@ -3154,6 +3193,11 @@ void PeerManagerImpl::ProcessBlock(CNode& node, const std::shared_ptr<const CBlo m_chainman.ProcessNewBlock(block, force_processing, min_pow_checked, &new_block); if (new_block) { node.m_last_block_time = GetTime<std::chrono::seconds>(); + // In case this block came from a different peer than we requested + // from, we can erase the block request now anyway (as we just stored + // this block to disk). + LOCK(cs_main); + RemoveBlockRequest(block->GetHash(), std::nullopt); } else { LOCK(cs_main); mapBlockSource.erase(block->GetHash()); @@ -3378,8 +3422,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // If the peer is old enough to have the old alert system, send it the final alert. if (greatest_common_version <= 70012) { - DataStream finalAlert{ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50")}; - m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make("alert", finalAlert)); + const auto finalAlert{ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50")}; + m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make("alert", Span{finalAlert})); } // Feeler connections exist only to verify if address is online. @@ -3874,7 +3918,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (pindex->nHeight >= m_chainman.ActiveChain().Height() - MAX_BLOCKTXN_DEPTH) { CBlock block; - bool ret = ReadBlockFromDisk(block, pindex, m_chainparams.GetConsensus()); + const bool ret{m_chainman.m_blockman.ReadBlockFromDisk(block, *pindex)}; assert(ret); SendBlockTransactions(pfrom, *peer, block, req); @@ -4262,15 +4306,27 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, nodestate->m_last_block_announcement = GetTime(); } - std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); - bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); - if (pindex->nStatus & BLOCK_HAVE_DATA) // Nothing to do here return; + auto range_flight = mapBlocksInFlight.equal_range(pindex->GetBlockHash()); + size_t already_in_flight = std::distance(range_flight.first, range_flight.second); + bool requested_block_from_this_peer{false}; + + // Multimap ensures ordering of outstanding requests. It's either empty or first in line. + bool first_in_flight = already_in_flight == 0 || (range_flight.first->second.first == pfrom.GetId()); + + while (range_flight.first != range_flight.second) { + if (range_flight.first->second.first == pfrom.GetId()) { + requested_block_from_this_peer = true; + break; + } + range_flight.first++; + } + if (pindex->nChainWork <= m_chainman.ActiveChain().Tip()->nChainWork || // We know something better pindex->nTx != 0) { // We had this block at some point, but pruned it - if (fAlreadyInFlight) { + if (requested_block_from_this_peer) { // We requested this block for some reason, but our mempool will probably be useless // so we just grab the block via normal getdata std::vector<CInv> vInv(1); @@ -4281,15 +4337,15 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } // If we're not close to tip yet, give up and let parallel block fetch work its magic - if (!fAlreadyInFlight && !CanDirectFetch()) { + if (!already_in_flight && !CanDirectFetch()) { return; } // We want to be a bit conservative just to be extra careful about DoS // possibilities in compact block processing... if (pindex->nHeight <= m_chainman.ActiveChain().Height() + 2) { - if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || - (fAlreadyInFlight && blockInFlightIt->second.first == pfrom.GetId())) { + if ((already_in_flight < MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK && nodestate->vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || + requested_block_from_this_peer) { std::list<QueuedBlock>::iterator* queuedBlockIt = nullptr; if (!BlockRequested(pfrom.GetId(), *pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) @@ -4304,14 +4360,19 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; ReadStatus status = partialBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status == READ_STATUS_INVALID) { - RemoveBlockRequest(pindex->GetBlockHash()); // Reset in-flight state in case Misbehaving does not result in a disconnect + RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect Misbehaving(*peer, 100, "invalid compact block"); return; } else if (status == READ_STATUS_FAILED) { - // Duplicate txindexes, the block is now in-flight, so just request it - std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), blockhash); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + if (first_in_flight) { + // Duplicate txindexes, the block is now in-flight, so just request it + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), blockhash); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + } else { + // Give up for this peer and wait for other peer(s) + RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); + } return; } @@ -4326,9 +4387,24 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, txn.blockhash = blockhash; blockTxnMsg << txn; fProcessBLOCKTXN = true; - } else { + } else if (first_in_flight) { + // We will try to round-trip any compact blocks we get on failure, + // as long as it's first... + req.blockhash = pindex->GetBlockHash(); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + } else if (pfrom.m_bip152_highbandwidth_to && + (!pfrom.IsInboundConn() || + IsBlockRequestedFromOutbound(blockhash) || + already_in_flight < MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK - 1)) { + // ... or it's a hb relay peer and: + // - peer is outbound, or + // - we already have an outbound attempt in flight(so we'll take what we can get), or + // - it's not the final parallel download slot (which we may reserve for first outbound) req.blockhash = pindex->GetBlockHash(); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + } else { + // Give up for this peer and wait for other peer(s) + RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); } } else { // This block is either already in flight from a different @@ -4349,7 +4425,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } } } else { - if (fAlreadyInFlight) { + if (requested_block_from_this_peer) { // We requested this block, but its far into the future, so our // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); @@ -4399,7 +4475,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // process from some other peer. We do this after calling // ProcessNewBlock so that a malleated cmpctblock announcement // can't be used to interfere with block relay. - RemoveBlockRequest(pblock->GetHash()); + RemoveBlockRequest(pblock->GetHash(), std::nullopt); } } return; @@ -4421,24 +4497,44 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, { LOCK(cs_main); - std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); - if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || - it->second.first != pfrom.GetId()) { + auto range_flight = mapBlocksInFlight.equal_range(resp.blockhash); + size_t already_in_flight = std::distance(range_flight.first, range_flight.second); + bool requested_block_from_this_peer{false}; + + // Multimap ensures ordering of outstanding requests. It's either empty or first in line. + bool first_in_flight = already_in_flight == 0 || (range_flight.first->second.first == pfrom.GetId()); + + while (range_flight.first != range_flight.second) { + auto [node_id, block_it] = range_flight.first->second; + if (node_id == pfrom.GetId() && block_it->partialBlock) { + requested_block_from_this_peer = true; + break; + } + range_flight.first++; + } + + if (!requested_block_from_this_peer) { LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom.GetId()); return; } - PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + PartiallyDownloadedBlock& partialBlock = *range_flight.first->second.second->partialBlock; ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn); if (status == READ_STATUS_INVALID) { - RemoveBlockRequest(resp.blockhash); // Reset in-flight state in case Misbehaving does not result in a disconnect + RemoveBlockRequest(resp.blockhash, pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect Misbehaving(*peer, 100, "invalid compact block/non-matching block transactions"); return; } else if (status == READ_STATUS_FAILED) { - // Might have collided, fall back to getdata now :( - std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(*peer), resp.blockhash)); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + if (first_in_flight) { + // Might have collided, fall back to getdata now :( + std::vector<CInv> invs; + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(*peer), resp.blockhash)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + } else { + RemoveBlockRequest(resp.blockhash, pfrom.GetId()); + LogPrint(BCLog::NET, "Peer %d sent us a compact block but it failed to reconstruct, waiting on first download to complete\n", pfrom.GetId()); + return; + } } else { // Block is either okay, or possibly we received // READ_STATUS_CHECKBLOCK_FAILED. @@ -4457,7 +4553,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // though the block was successfully read, and rely on the // handling in ProcessNewBlock to ensure the block index is // updated, etc. - RemoveBlockRequest(resp.blockhash); // it is now an empty pointer + RemoveBlockRequest(resp.blockhash, pfrom.GetId()); // it is now an empty pointer fBlockRead = true; // mapBlockSource is used for potentially punishing peers and // updating which peers send us compact blocks, so the race @@ -4546,7 +4642,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Always process the block if we requested it, since we may // need it even when it's not a candidate for a new best tip. forceProcessing = IsBlockRequested(hash); - RemoveBlockRequest(hash); + RemoveBlockRequest(hash, pfrom.GetId()); // mapBlockSource is only used for punishing peers and setting // which peers send us compact blocks, so the race between here and // cs_main in ProcessNewBlock is fine. @@ -4600,6 +4696,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::MEMPOOL) { + // Only process received mempool messages if we advertise NODE_BLOOM + // or if the peer has mempool permissions. if (!(peer->m_our_services & NODE_BLOOM) && !pfrom.HasPermission(NetPermissionFlags::Mempool)) { if (!pfrom.HasPermission(NetPermissionFlags::NoBan)) @@ -5029,14 +5127,14 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now) // valid headers chain with at least as much work as our tip. CNodeState *node_state = State(pnode->GetId()); if (node_state == nullptr || - (now - pnode->m_connected >= MINIMUM_CONNECT_TIME && node_state->nBlocksInFlight == 0)) { + (now - pnode->m_connected >= MINIMUM_CONNECT_TIME && node_state->vBlocksInFlight.empty())) { pnode->fDisconnect = true; LogPrint(BCLog::NET, "disconnecting extra block-relay-only peer=%d (last block received at time %d)\n", pnode->GetId(), count_seconds(pnode->m_last_block_time)); return true; } else { LogPrint(BCLog::NET, "keeping block-relay-only peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", - pnode->GetId(), count_seconds(pnode->m_connected), node_state->nBlocksInFlight); + pnode->GetId(), count_seconds(pnode->m_connected), node_state->vBlocksInFlight.size()); } return false; }); @@ -5076,13 +5174,13 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now) // Also don't disconnect any peer we're trying to download a // block from. CNodeState &state = *State(pnode->GetId()); - if (now - pnode->m_connected > MINIMUM_CONNECT_TIME && state.nBlocksInFlight == 0) { + if (now - pnode->m_connected > MINIMUM_CONNECT_TIME && state.vBlocksInFlight.empty()) { LogPrint(BCLog::NET, "disconnecting extra outbound peer=%d (last block announcement received at time %d)\n", pnode->GetId(), oldest_block_announcement); pnode->fDisconnect = true; return true; } else { LogPrint(BCLog::NET, "keeping outbound peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", - pnode->GetId(), count_seconds(pnode->m_connected), state.nBlocksInFlight); + pnode->GetId(), count_seconds(pnode->m_connected), state.vBlocksInFlight.size()); return false; } }); @@ -5529,7 +5627,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) m_connman.PushMessage(pto, std::move(cached_cmpctblock_msg.value())); } else { CBlock block; - bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); + const bool ret{m_chainman.m_blockman.ReadBlockFromDisk(block, *pBestIndex)}; assert(ret); CBlockHeaderAndShortTxIDs cmpctblock{block}; m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); @@ -5663,7 +5761,9 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // especially since we have many peers and some will draw much shorter delays. unsigned int nRelayedTransactions = 0; LOCK(tx_relay->m_bloom_filter_mutex); - while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) { + size_t broadcast_max{INVENTORY_BROADCAST_MAX + (tx_relay->m_tx_inventory_to_send.size()/1000)*5}; + broadcast_max = std::min<size_t>(1000, broadcast_max); + while (!vInvTx.empty() && nRelayedTransactions < broadcast_max) { // Fetch the top element from the heap std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); std::set<uint256>::iterator it = vInvTx.back(); @@ -5682,7 +5782,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) continue; } auto txid = txinfo.tx->GetHash(); - auto wtxid = txinfo.tx->GetWitnessHash(); // Peer told you to not send transactions at that feerate? Don't bother sending it. if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; @@ -5692,24 +5791,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) tx_relay->m_recently_announced_invs.insert(hash); vInv.push_back(inv); nRelayedTransactions++; - { - // Expire old relay messages - while (!g_relay_expiration.empty() && g_relay_expiration.front().first < current_time) - { - mapRelay.erase(g_relay_expiration.front().second); - g_relay_expiration.pop_front(); - } - - auto ret = mapRelay.emplace(txid, std::move(txinfo.tx)); - if (ret.second) { - g_relay_expiration.emplace_back(current_time + RELAY_TX_CACHE_TIME, ret.first); - } - // Add wtxid-based lookup into mapRelay as well, so that peers can request by wtxid - auto ret2 = mapRelay.emplace(wtxid, ret.first->second); - if (ret2.second) { - g_relay_expiration.emplace_back(current_time + RELAY_TX_CACHE_TIME, ret2.first); - } - } if (vInv.size() == MAX_INV_SZ) { m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); @@ -5735,7 +5816,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. - LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->GetId()); + LogPrintf("Peer=%d%s is stalling block download, disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; // Increase timeout for the next peer so that we don't disconnect multiple peers if our own // bandwidth is insufficient. @@ -5754,7 +5835,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); int nOtherPeersWithValidatedDownloads = m_peers_downloading_from - 1; if (current_time > state.m_downloading_since + std::chrono::seconds{consensusParams.nPowTargetSpacing} * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { - LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->GetId()); + LogPrintf("Timeout downloading block %s from peer=%d%s, disconnecting\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; return true; } @@ -5770,11 +5851,11 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // disconnect our sync peer for stalling; we have bigger // problems if we can't get any outbound peers. if (!pto->HasPermission(NetPermissionFlags::NoBan)) { - LogPrintf("Timeout downloading headers from peer=%d, disconnecting\n", pto->GetId()); + LogPrintf("Timeout downloading headers from peer=%d%s, disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; return true; } else { - LogPrintf("Timeout downloading headers from noban peer=%d, not disconnecting\n", pto->GetId()); + LogPrintf("Timeout downloading headers from noban peer=%d%s, not disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); // Reset the headers sync state so that we have a // chance to try downloading from a different peer. // Note: this will also result in at least one more @@ -5800,10 +5881,10 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Message: getdata (blocks) // std::vector<CInv> vGetData; - if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { std::vector<const CBlockIndex*> vToDownload; NodeId staller = -1; - FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); + FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.vBlocksInFlight.size(), vToDownload, staller); for (const CBlockIndex *pindex : vToDownload) { uint32_t nFetchFlags = GetFetchFlags(*peer); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); @@ -5811,7 +5892,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), pindex->nHeight, pto->GetId()); } - if (state.nBlocksInFlight == 0 && staller != -1) { + if (state.vBlocksInFlight.empty() && staller != -1) { if (State(staller)->m_stalling_since == 0us) { State(staller)->m_stalling_since = current_time; LogPrint(BCLog::NET, "Stall started peer=%d\n", staller); diff --git a/src/net_processing.h b/src/net_processing.h index af9a02139b..deebb24c94 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -22,6 +22,8 @@ static const bool DEFAULT_PEERBLOOMFILTERS = false; static const bool DEFAULT_PEERBLOCKFILTERS = false; /** Threshold for marking a node to be discouraged, e.g. disconnected and added to the discouragement filter. */ static const int DISCOURAGEMENT_THRESHOLD{100}; +/** Maximum number of outstanding CMPCTBLOCK requests for the same block. */ +static const unsigned int MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK = 3; struct CNodeStateStats { int nSyncHeight = -1; diff --git a/src/netbase.cpp b/src/netbase.cpp index f39a3635f4..8f6f92ea7d 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -36,8 +36,8 @@ static Proxy nameProxy GUARDED_BY(g_proxyinfo_mutex); int nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; bool fNameLookup = DEFAULT_NAME_LOOKUP; -// Need ample time for negotiation for very slow proxies such as Tor (milliseconds) -int g_socks5_recv_timeout = 20 * 1000; +// Need ample time for negotiation for very slow proxies such as Tor +std::chrono::milliseconds g_socks5_recv_timeout = 20s; static std::atomic<bool> interruptSocks5Recv(false); std::vector<CNetAddr> WrappedGetAddrInfo(const std::string& name, bool allow_lookup) @@ -132,14 +132,9 @@ std::vector<std::string> GetNetworkNames(bool append_unroutable) return names; } -static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) +static std::vector<CNetAddr> LookupIntern(const std::string& name, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - vIP.clear(); - - if (!ContainsNoNUL(name)) { - return false; - } - + if (!ContainsNoNUL(name)) return {}; { CNetAddr addr; // From our perspective, onion addresses are not hostnames but rather @@ -148,83 +143,65 @@ static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, un // getaddrinfo to decode them and it wouldn't make sense to resolve // them, we return a network address representing it instead. See // CNetAddr::SetSpecial(const std::string&) for more details. - if (addr.SetSpecial(name)) { - vIP.push_back(addr); - return true; - } + if (addr.SetSpecial(name)) return {addr}; } + std::vector<CNetAddr> addresses; + for (const CNetAddr& resolved : dns_lookup_function(name, fAllowLookup)) { - if (nMaxSolutions > 0 && vIP.size() >= nMaxSolutions) { + if (nMaxSolutions > 0 && addresses.size() >= nMaxSolutions) { break; } /* Never allow resolving to an internal address. Consider any such result invalid */ if (!resolved.IsInternal()) { - vIP.push_back(resolved); + addresses.push_back(resolved); } } - return (vIP.size() > 0); + return addresses; } -bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) +std::vector<CNetAddr> LookupHost(const std::string& name, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ContainsNoNUL(name)) { - return false; - } + if (!ContainsNoNUL(name)) return {}; std::string strHost = name; - if (strHost.empty()) - return false; + if (strHost.empty()) return {}; if (strHost.front() == '[' && strHost.back() == ']') { strHost = strHost.substr(1, strHost.size() - 2); } - return LookupIntern(strHost, vIP, nMaxSolutions, fAllowLookup, dns_lookup_function); + return LookupIntern(strHost, nMaxSolutions, fAllowLookup, dns_lookup_function); } -bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSLookupFn dns_lookup_function) +std::optional<CNetAddr> LookupHost(const std::string& name, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ContainsNoNUL(name)) { - return false; - } - std::vector<CNetAddr> vIP; - LookupHost(name, vIP, 1, fAllowLookup, dns_lookup_function); - if(vIP.empty()) - return false; - addr = vIP.front(); - return true; + const std::vector<CNetAddr> addresses{LookupHost(name, 1, fAllowLookup, dns_lookup_function)}; + return addresses.empty() ? std::nullopt : std::make_optional(addresses.front()); } -bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function) +std::vector<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function) { if (name.empty() || !ContainsNoNUL(name)) { - return false; + return {}; } uint16_t port{portDefault}; std::string hostname; SplitHostPort(name, port, hostname); - std::vector<CNetAddr> vIP; - bool fRet = LookupIntern(hostname, vIP, nMaxSolutions, fAllowLookup, dns_lookup_function); - if (!fRet) - return false; - vAddr.resize(vIP.size()); - for (unsigned int i = 0; i < vIP.size(); i++) - vAddr[i] = CService(vIP[i], port); - return true; + const std::vector<CNetAddr> addresses{LookupIntern(hostname, nMaxSolutions, fAllowLookup, dns_lookup_function)}; + if (addresses.empty()) return {}; + std::vector<CService> services; + services.reserve(addresses.size()); + for (const auto& addr : addresses) + services.emplace_back(addr, port); + return services; } -bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function) +std::optional<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ContainsNoNUL(name)) { - return false; - } - std::vector<CService> vService; - bool fRet = Lookup(name, vService, portDefault, fAllowLookup, 1, dns_lookup_function); - if (!fRet) - return false; - addr = vService[0]; - return true; + const std::vector<CService> services{Lookup(name, portDefault, fAllowLookup, 1, dns_lookup_function)}; + + return services.empty() ? std::nullopt : std::make_optional(services.front()); } CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupFn dns_lookup_function) @@ -232,12 +209,9 @@ CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupF if (!ContainsNoNUL(name)) { return {}; } - CService addr; // "1.2:345" will fail to resolve the ip, but will still set the port. // If the ip fails to resolve, re-init the result. - if(!Lookup(name, addr, portDefault, false, dns_lookup_function)) - addr = CService(); - return addr; + return Lookup(name, portDefault, /*fAllowLookup=*/false, dns_lookup_function).value_or(CService{}); } /** SOCKS version */ @@ -296,7 +270,7 @@ enum class IntrRecvError { * * @param data The buffer where the read bytes should be stored. * @param len The number of bytes to read into the specified buffer. - * @param timeout The total timeout in milliseconds for this read. + * @param timeout The total timeout for this read. * @param sock The socket (has to be in non-blocking mode) from which to read bytes. * * @returns An IntrRecvError indicating the resulting status of this read. @@ -306,10 +280,10 @@ enum class IntrRecvError { * @see This function can be interrupted by calling InterruptSocks5(bool). * Sockets can be made non-blocking with Sock::SetNonBlocking(). */ -static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const Sock& sock) +static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, std::chrono::milliseconds timeout, const Sock& sock) { - int64_t curTime = GetTimeMillis(); - int64_t endTime = curTime + timeout; + auto curTime{Now<SteadyMilliseconds>()}; + const auto endTime{curTime + timeout}; while (len > 0 && curTime < endTime) { ssize_t ret = sock.Recv(data, len, 0); // Optimistically try the recv first if (ret > 0) { @@ -333,7 +307,7 @@ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, c } if (interruptSocks5Recv) return IntrRecvError::Interrupted; - curTime = GetTimeMillis(); + curTime = Now<SteadyMilliseconds>(); } return len == 0 ? IntrRecvError::OK : IntrRecvError::Timeout; } @@ -689,27 +663,27 @@ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) const size_t slash_pos{subnet_str.find_last_of('/')}; const std::string str_addr{subnet_str.substr(0, slash_pos)}; - CNetAddr addr; + const std::optional<CNetAddr> addr{LookupHost(str_addr, /*fAllowLookup=*/false)}; - if (LookupHost(str_addr, addr, /*fAllowLookup=*/false)) { + if (addr.has_value()) { if (slash_pos != subnet_str.npos) { const std::string netmask_str{subnet_str.substr(slash_pos + 1)}; uint8_t netmask; if (ParseUInt8(netmask_str, &netmask)) { // Valid number; assume CIDR variable-length subnet masking. - subnet_out = CSubNet{addr, netmask}; + subnet_out = CSubNet{addr.value(), netmask}; return subnet_out.IsValid(); } else { // Invalid number; try full netmask syntax. Never allow lookup for netmask. - CNetAddr full_netmask; - if (LookupHost(netmask_str, full_netmask, /*fAllowLookup=*/false)) { - subnet_out = CSubNet{addr, full_netmask}; + const std::optional<CNetAddr> full_netmask{LookupHost(netmask_str, /*fAllowLookup=*/false)}; + if (full_netmask.has_value()) { + subnet_out = CSubNet{addr.value(), full_netmask.value()}; return subnet_out.IsValid(); } } } else { // Single IP subnet (<ipv4>/32 or <ipv6>/128). - subnet_out = CSubNet{addr}; + subnet_out = CSubNet{addr.value()}; return subnet_out.IsValid(); } } diff --git a/src/netbase.h b/src/netbase.h index a43f22f240..1da4f5c51d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -105,24 +105,25 @@ extern DNSLookupFn g_dns_lookup; * @param name The string representing a host. Could be a name or a numerical * IP address (IPv6 addresses in their bracketed form are * allowed). - * @param[out] vIP The resulting network addresses to which the specified host - * string resolved. * - * @returns Whether or not the specified host string successfully resolved to - * any resulting network addresses. + * @returns The resulting network addresses to which the specified host + * string resolved. * - * @see Lookup(const std::string&, std::vector<CService>&, uint16_t, bool, unsigned int, DNSLookupFn) + * @see Lookup(const std::string&, uint16_t, bool, unsigned int, DNSLookupFn) * for additional parameter descriptions. */ -bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::vector<CNetAddr> LookupHost(const std::string& name, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a host string to its first corresponding network address. * - * @see LookupHost(const std::string&, std::vector<CNetAddr>&, uint16_t, bool, DNSLookupFn) + * @returns The resulting network address to which the specified host + * string resolved or std::nullopt if host does not resolve to an address. + * + * @see LookupHost(const std::string&, unsigned int, bool, DNSLookupFn) * for additional parameter descriptions. */ -bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::optional<CNetAddr> LookupHost(const std::string& name, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a service string to its corresponding service. @@ -132,8 +133,6 @@ bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSL * disambiguated bracketed form), optionally followed by a uint16_t port * number. (e.g. example.com:8333 or * [2001:db8:85a3:8d3:1319:8a2e:370:7348]:420) - * @param[out] vAddr The resulting services to which the specified service string - * resolved. * @param portDefault The default port for resulting services if not specified * by the service string. * @param fAllowLookup Whether or not hostname lookups are permitted. If yes, @@ -141,18 +140,18 @@ bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSL * @param nMaxSolutions The maximum number of results we want, specifying 0 * means "as many solutions as we get." * - * @returns Whether or not the service string successfully resolved to any - * resulting services. + * @returns The resulting services to which the specified service string + * resolved. */ -bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::vector<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a service string to its first corresponding service. * - * @see Lookup(const std::string&, std::vector<CService>&, uint16_t, bool, unsigned int, DNSLookupFn) + * @see Lookup(const std::string&, uint16_t, bool, unsigned int, DNSLookupFn) * for additional parameter descriptions. */ -bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::optional<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a service string with a numeric IP to its first corresponding @@ -160,7 +159,7 @@ bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool * * @returns The resulting CService if the resolution was successful, [::]:0 otherwise. * - * @see Lookup(const std::string&, std::vector<CService>&, uint16_t, bool, unsigned int, DNSLookupFn) + * @see Lookup(const std::string&, uint16_t, bool, unsigned int, DNSLookupFn) * for additional parameter descriptions. */ CService LookupNumeric(const std::string& name, uint16_t portDefault = 0, DNSLookupFn dns_lookup_function = g_dns_lookup); diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index 06a1934947..4b296db1b0 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -5,26 +5,35 @@ #include <node/blockmanager_args.h> #include <common/args.h> +#include <node/blockstorage.h> +#include <tinyformat.h> +#include <util/result.h> +#include <util/translation.h> #include <validation.h> +#include <cstdint> + namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts) +util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts) { // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg{args.GetIntArg("-prune", opts.prune_target)}; if (nPruneArg < 0) { - return _("Prune cannot be configured with a negative value."); + return util::Error{_("Prune cannot be configured with a negative value.")}; } uint64_t nPruneTarget{uint64_t(nPruneArg) * 1024 * 1024}; if (nPruneArg == 1) { // manual pruning: -prune=1 nPruneTarget = BlockManager::PRUNE_TARGET_MANUAL; } else if (nPruneTarget) { if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) { - return strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024); + return util::Error{strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)}; } } opts.prune_target = nPruneTarget; - return std::nullopt; + if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value; + if (auto value{args.GetBoolArg("-stopafterblockimport")}) opts.stop_after_block_import = *value; + + return {}; } } // namespace node diff --git a/src/node/blockmanager_args.h b/src/node/blockmanager_args.h index e657c6bb45..de9fe83a5c 100644 --- a/src/node/blockmanager_args.h +++ b/src/node/blockmanager_args.h @@ -7,14 +7,12 @@ #define BITCOIN_NODE_BLOCKMANAGER_ARGS_H #include <node/blockstorage.h> - -#include <optional> +#include <util/result.h> class ArgsManager; -struct bilingual_str; namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts); +[[nodiscard]] util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts); } // namespace node #endif // BITCOIN_NODE_BLOCKMANAGER_ARGS_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index abd71ca50b..1368ae6f6d 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -6,7 +6,6 @@ #include <chain.h> #include <clientversion.h> -#include <common/args.h> #include <consensus/validation.h> #include <flatfile.h> #include <hash.h> @@ -18,9 +17,9 @@ #include <signet.h> #include <streams.h> #include <undo.h> +#include <util/batchpriority.h> #include <util/fs.h> #include <util/syscall_sandbox.h> -#include <util/system.h> #include <validation.h> #include <map> @@ -28,6 +27,7 @@ namespace node { std::atomic_bool fReindex(false); +std::atomic_bool g_indexes_ready_to_sync{false}; bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const { @@ -53,10 +53,6 @@ bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CB return pa->nHeight < pb->nHeight; } -static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); -static FlatFileSeq BlockFileSeq(); -static FlatFileSeq UndoFileSeq(); - std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices() { AssertLockHeld(cs_main); @@ -253,9 +249,9 @@ CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) return pindex; } -bool BlockManager::LoadBlockIndex(const Consensus::Params& consensus_params) +bool BlockManager::LoadBlockIndex() { - if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { + if (!m_block_tree_db->LoadBlockIndexGuts(GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { return false; } @@ -318,9 +314,9 @@ bool BlockManager::WriteBlockIndexDB() return true; } -bool BlockManager::LoadBlockIndexDB(const Consensus::Params& consensus_params) +bool BlockManager::LoadBlockIndexDB() { - if (!LoadBlockIndex(consensus_params)) { + if (!LoadBlockIndex()) { return false; } @@ -423,7 +419,7 @@ const CBlockIndex* BlockManager::GetFirstStoredBlock(const CBlockIndex& start_bl // rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info // is in sync with what's actually on disk by the time we start downloading, so that pruning // works correctly. -void CleanupBlockRevFiles() +void BlockManager::CleanupBlockRevFiles() const { std::map<std::string, fs::path> mapBlockFiles; @@ -431,8 +427,7 @@ void CleanupBlockRevFiles() // Remove the rev files immediately and insert the blk file paths into an // ordered map keyed by block file index. LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); - const fs::path& blocksdir = gArgs.GetBlocksDirPath(); - for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { + for (fs::directory_iterator it(m_opts.blocks_dir); it != fs::directory_iterator(); it++) { const std::string path = fs::PathToString(it->path().filename()); if (fs::is_regular_file(*it) && path.length() == 12 && @@ -467,7 +462,7 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n) return &m_blockfile_info.at(n); } -static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) const { // Open history file to append AutoFile fileout{OpenUndoFile(pos)}; @@ -496,9 +491,9 @@ static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const return true; } -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) +bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const { - const FlatFilePos pos{WITH_LOCK(::cs_main, return pindex->GetUndoPos())}; + const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())}; if (pos.IsNull()) { return error("%s: no undo data available", __func__); @@ -514,7 +509,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) uint256 hashChecksum; HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a try { - verifier << pindex->pprev->GetBlockHash(); + verifier << index.pprev->GetBlockHash(); verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { @@ -570,7 +565,7 @@ uint64_t BlockManager::CalculateCurrentUsage() return retval; } -void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) +void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const { std::error_code ec; for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { @@ -583,28 +578,28 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) } } -static FlatFileSeq BlockFileSeq() +FlatFileSeq BlockManager::BlockFileSeq() const { - return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); + return FlatFileSeq(m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); } -static FlatFileSeq UndoFileSeq() +FlatFileSeq BlockManager::UndoFileSeq() const { - return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); + return FlatFileSeq(m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE); } -FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) +FILE* BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const { return BlockFileSeq().Open(pos, fReadOnly); } /** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) +FILE* BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const { return UndoFileSeq().Open(pos, fReadOnly); } -fs::path GetBlockPosFilename(const FlatFilePos& pos) +fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const { return BlockFileSeq().FileName(pos); } @@ -623,7 +618,7 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE}; // Use smaller blockfiles in test-only -fastprune mode - but avoid // the possibility of having a block not fit into the block file. - if (gArgs.GetBoolArg("-fastprune", false)) { + if (m_opts.fast_prune) { max_blockfile_size = 0x10000; // 64kiB if (nAddSize >= max_blockfile_size) { // dynamically adjust the blockfile size to be larger than the added size @@ -697,7 +692,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP return true; } -static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) +bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); @@ -720,16 +715,16 @@ static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessa return true; } -bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block) { AssertLockHeld(::cs_main); // Write undo information to disk - if (pindex->GetUndoPos().IsNull()) { + if (block.GetUndoPos().IsNull()) { FlatFilePos _pos; - if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) { + if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) { return error("ConnectBlock(): FindUndoPos failed"); } - if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) { + if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash(), GetParams().MessageStart())) { return AbortNode(state, "Failed to write undo data"); } // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) @@ -737,20 +732,20 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid // in the block file info as below; note that this does not catch the case where the undo writes are keeping up // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in // the FindBlockPos function - if (_pos.nFile < m_last_blockfile && static_cast<uint32_t>(pindex->nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { + if (_pos.nFile < m_last_blockfile && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { FlushUndoFile(_pos.nFile, true); } // update nUndoPos in block index - pindex->nUndoPos = _pos.nPos; - pindex->nStatus |= BLOCK_HAVE_UNDO; - m_dirty_blockindex.insert(pindex); + block.nUndoPos = _pos.nPos; + block.nStatus |= BLOCK_HAVE_UNDO; + m_dirty_blockindex.insert(&block); } return true; } -bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) +bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const { block.SetNull(); @@ -768,33 +763,33 @@ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::P } // Check the header - if (!CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) { + if (!CheckProofOfWork(block.GetHash(), block.nBits, GetConsensus())) { return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); } // Signet only: check block solution - if (consensusParams.signet_blocks && !CheckSignetBlockSolution(block, consensusParams)) { + if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) { return error("ReadBlockFromDisk: Errors in block solution at %s", pos.ToString()); } return true; } -bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) +bool BlockManager::ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const { - const FlatFilePos block_pos{WITH_LOCK(cs_main, return pindex->GetBlockPos())}; + const FlatFilePos block_pos{WITH_LOCK(cs_main, return index.GetBlockPos())}; - if (!ReadBlockFromDisk(block, block_pos, consensusParams)) { + if (!ReadBlockFromDisk(block, block_pos)) { return false; } - if (block.GetHash() != pindex->GetBlockHash()) { + if (block.GetHash() != index.GetBlockHash()) { return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", - pindex->ToString(), block_pos.ToString()); + index.ToString(), block_pos.ToString()); } return true; } -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) +bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) const { FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header @@ -829,7 +824,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c return true; } -FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) +FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; @@ -847,7 +842,7 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CCha return FlatFilePos(); } if (!position_known) { - if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { + if (!WriteBlockToDisk(block, blockPos, GetParams().MessageStart())) { AbortNode("Failed to write block"); return FlatFilePos(); } @@ -872,7 +867,7 @@ public: } }; -void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args, const fs::path& mempool_path) +void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const fs::path& mempool_path) { SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); @@ -888,10 +883,10 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; while (true) { FlatFilePos pos(nFile, 0); - if (!fs::exists(GetBlockPosFilename(pos))) { + if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) { break; // No block files left to reindex } - FILE* file = OpenBlockFile(pos, true); + FILE* file = chainman.m_blockman.OpenBlockFile(pos, true); if (!file) { break; // This error is logged in OpenBlockFile } @@ -933,18 +928,18 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { BlockValidationState state; if (!chainstate->ActivateBestChain(state, nullptr)) { - LogPrintf("Failed to connect best block (%s)\n", state.ToString()); - StartShutdown(); + AbortNode(strprintf("Failed to connect best block (%s)", state.ToString())); return; } } - if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { + if (chainman.m_blockman.StopAfterBlockImport()) { LogPrintf("Stopping after block import\n"); StartShutdown(); return; } } // End scope of ImportingNow chainman.ActiveChainstate().LoadMempool(mempool_path); + g_indexes_ready_to_sync = true; } } // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 3eb27cc72d..a1ebb0df0a 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -8,6 +8,7 @@ #include <attributes.h> #include <chain.h> #include <kernel/blockmanager_opts.h> +#include <kernel/chainparams.h> #include <kernel/cs_main.h> #include <protocol.h> #include <sync.h> @@ -19,7 +20,6 @@ #include <unordered_map> #include <vector> -class ArgsManager; class BlockValidationState; class CBlock; class CBlockFileInfo; @@ -35,7 +35,6 @@ struct Params; } namespace node { -static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB @@ -48,6 +47,7 @@ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); extern std::atomic_bool fReindex; +extern std::atomic_bool g_indexes_ready_to_sync; // Because validation code takes pointers to the map's CBlockIndex objects, if // we ever switch to another associative container, we need to either use a @@ -81,18 +81,28 @@ class BlockManager friend ChainstateManager; private: + const CChainParams& GetParams() const { return m_opts.chainparams; } + const Consensus::Params& GetConsensus() const { return m_opts.chainparams.GetConsensus(); } /** * Load the blocktree off disk and into memory. Populate certain metadata * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral * collections like m_dirty_blockindex. */ - bool LoadBlockIndex(const Consensus::Params& consensus_params) + bool LoadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(cs_main); void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); void FlushUndoFile(int block_file, bool finalize = false); bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown); bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); + FlatFileSeq BlockFileSeq() const; + FlatFileSeq UndoFileSeq() const; + + FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; + + bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const; + bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) const; + /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height); @@ -162,7 +172,7 @@ public: std::unique_ptr<CBlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main); bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - bool LoadBlockIndexDB(const Consensus::Params& consensus_params) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * Remove any pruned block & undo files that are still on disk. @@ -184,11 +194,11 @@ public: /** Get block file info entry for one block file */ CBlockFileInfo* GetBlockFileInfo(size_t n); - bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) + bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ - FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); + FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const FlatFilePos* dbp); /** Whether running in -prune mode. */ [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; } @@ -199,6 +209,8 @@ public: [[nodiscard]] bool LoadingBlocks() const { return m_importing || fReindex; } + [[nodiscard]] bool StopAfterBlockImport() const { return m_opts.stop_after_block_import; } + /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); @@ -216,28 +228,29 @@ public: //! Create or update a prune lock identified by its name void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); -}; -void CleanupBlockRevFiles(); + /** Open a block file (blk?????.dat) */ + FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const; -/** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false); -/** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const FlatFilePos& pos); + /** Translation to a filesystem path */ + fs::path GetBlockPosFilename(const FlatFilePos& pos) const; -/** - * Actually unlink the specified files - */ -void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); + /** + * Actually unlink the specified files + */ + void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const; -/** Functions for disk access for blocks */ -bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); -bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start); + /** Functions for disk access for blocks */ + bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const; + bool ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const; + bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) const; -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); + bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const; + + void CleanupBlockRevFiles() const; +}; -void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args, const fs::path& mempool_path); +void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const fs::path& mempool_path); } // namespace node #endif // BITCOIN_NODE_BLOCKSTORAGE_H diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index f60ff83a0d..8f997b0594 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -51,7 +51,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( pblocktree->WriteReindexing(true); //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (options.prune) { - CleanupBlockRevFiles(); + chainman.m_blockman.CleanupBlockRevFiles(); } } @@ -253,7 +253,7 @@ ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const C "Only rebuild the block database if you are sure that your computer's date and time are correct")}; } - VerifyDBResult result = CVerifyDB().VerifyDB( + VerifyDBResult result = CVerifyDB(chainman.GetNotifications()).VerifyDB( *chainstate, chainman.GetConsensus(), chainstate->CoinsDB(), options.check_level, options.check_blocks); diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index b97344c9aa..a7f7303348 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -11,16 +11,16 @@ #include <node/database_args.h> #include <tinyformat.h> #include <uint256.h> +#include <util/result.h> #include <util/strencodings.h> #include <util/translation.h> #include <validation.h> #include <chrono> -#include <optional> #include <string> namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) +util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) { if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; @@ -28,7 +28,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, Chains if (auto value{args.GetArg("-minimumchainwork")}) { if (!IsHexNumber(*value)) { - return strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), *value); + return util::Error{strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), *value)}; } opts.minimum_chain_work = UintToArith256(uint256S(*value)); } @@ -37,10 +37,12 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, Chains if (auto value{args.GetIntArg("-maxtipage")}) opts.max_tip_age = std::chrono::seconds{*value}; + if (auto value{args.GetIntArg("-stopatheight")}) opts.stop_at_height = *value; + ReadDatabaseArgs(args, opts.block_tree_db); ReadDatabaseArgs(args, opts.coins_db); ReadCoinsViewArgs(args, opts.coins_view); - return std::nullopt; + return {}; } } // namespace node diff --git a/src/node/chainstatemanager_args.h b/src/node/chainstatemanager_args.h index 6c46b998f2..701515953e 100644 --- a/src/node/chainstatemanager_args.h +++ b/src/node/chainstatemanager_args.h @@ -5,15 +5,13 @@ #ifndef BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H #define BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H +#include <util/result.h> #include <validation.h> -#include <optional> - class ArgsManager; -struct bilingual_str; namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); +[[nodiscard]] util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); } // namespace node #endif // BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H diff --git a/src/node/context.cpp b/src/node/context.cpp index af59ab932b..ca56fa0b86 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -11,6 +11,7 @@ #include <net.h> #include <net_processing.h> #include <netgroup.h> +#include <node/kernel_notifications.h> #include <policy/fees.h> #include <scheduler.h> #include <txmempool.h> diff --git a/src/node/context.h b/src/node/context.h index 84f4053c84..91b68fa5bb 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -7,7 +7,9 @@ #include <kernel/context.h> +#include <atomic> #include <cassert> +#include <cstdlib> #include <functional> #include <memory> #include <vector> @@ -30,6 +32,8 @@ class WalletLoader; } // namespace interfaces namespace node { +class KernelNotifications; + //! NodeContext struct containing references to chain state and connection //! state. //! @@ -62,6 +66,8 @@ struct NodeContext { interfaces::WalletLoader* wallet_loader{nullptr}; std::unique_ptr<CScheduler> scheduler; std::function<void()> rpc_interruption_point = [] {}; + std::unique_ptr<KernelNotifications> notifications; + std::atomic<int> exit_status{EXIT_SUCCESS}; //! Declare default constructor and destructor that are not inline, so code //! instantiating the NodeContext struct doesn't need to #include class diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 5fcace833f..94b607b1de 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -89,10 +89,11 @@ public: void initLogging() override { InitLogging(args()); } void initParameterInteraction() override { InitParameterInteraction(args()); } bilingual_str getWarnings() override { return GetWarnings(true); } + int getExitStatus() override { return Assert(m_context)->exit_status.load(); } uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override { - if (!AppInitBasicSetup(args())) return false; + if (!AppInitBasicSetup(args(), Assert(context())->exit_status)) return false; if (!AppInitParameterInteraction(args(), /*use_syscall_sandbox=*/false)) return false; m_context->kernel = std::make_unique<kernel::Context>(); @@ -105,7 +106,10 @@ public: } bool appInitMain(interfaces::BlockAndHeaderTipInfo* tip_info) override { - return AppInitMain(*m_context, tip_info); + if (AppInitMain(*m_context, tip_info)) return true; + // Error during initialization, set exit status before continue + m_context->exit_status.store(EXIT_FAILURE); + return false; } void appShutdown() override { @@ -125,17 +129,17 @@ public: bool isSettingIgnored(const std::string& name) override { bool ignored = false; - args().LockSettings([&](util::Settings& settings) { - if (auto* options = util::FindKey(settings.command_line_options, name)) { + args().LockSettings([&](common::Settings& settings) { + if (auto* options = common::FindKey(settings.command_line_options, name)) { ignored = !options->empty(); } }); return ignored; } - util::SettingsValue getPersistentSetting(const std::string& name) override { return args().GetPersistentSetting(name); } - void updateRwSetting(const std::string& name, const util::SettingsValue& value) override + common::SettingsValue getPersistentSetting(const std::string& name) override { return args().GetPersistentSetting(name); } + void updateRwSetting(const std::string& name, const common::SettingsValue& value) override { - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { if (value.isNull()) { settings.rw_settings.erase(name); } else { @@ -144,9 +148,9 @@ public: }); args().WriteSettingsFile(); } - void forceSetting(const std::string& name, const util::SettingsValue& value) override + void forceSetting(const std::string& name, const common::SettingsValue& value) override { - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { if (value.isNull()) { settings.forced_settings.erase(name); } else { @@ -157,7 +161,7 @@ public: void resetSettings() override { args().WriteSettingsFile(/*errors=*/nullptr, /*backup=*/true); - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { settings.rw_settings.clear(); }); args().WriteSettingsFile(); @@ -239,7 +243,7 @@ public: std::vector<ExternalSigner> signers = {}; const std::string command = args().GetArg("-signer", ""); if (command == "") return {}; - ExternalSigner::Enumerate(command, signers, Params().NetworkIDString()); + ExternalSigner::Enumerate(command, signers, Params().GetChainTypeString()); std::vector<std::unique_ptr<interfaces::ExternalSigner>> result; result.reserve(signers.size()); for (auto& signer : signers) { @@ -394,7 +398,7 @@ public: NodeContext* m_context{nullptr}; }; -bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock, const CChain& active) +bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock, const CChain& active, const BlockManager& blockman) { if (!index) return false; if (block.m_hash) *block.m_hash = index->GetBlockHash(); @@ -404,10 +408,10 @@ bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<Rec if (block.m_mtp_time) *block.m_mtp_time = index->GetMedianTimePast(); if (block.m_in_active_chain) *block.m_in_active_chain = active[index->nHeight] == index; if (block.m_locator) { *block.m_locator = GetLocator(index); } - if (block.m_next_block) FillBlock(active[index->nHeight] == index ? active[index->nHeight + 1] : nullptr, *block.m_next_block, lock, active); + if (block.m_next_block) FillBlock(active[index->nHeight] == index ? active[index->nHeight + 1] : nullptr, *block.m_next_block, lock, active, blockman); if (block.m_data) { REVERSE_LOCK(lock); - if (!ReadBlockFromDisk(*block.m_data, index, Params().GetConsensus())) block.m_data->SetNull(); + if (!blockman.ReadBlockFromDisk(*block.m_data, *index)) block.m_data->SetNull(); } block.found = true; return true; @@ -557,13 +561,13 @@ public: bool findBlock(const uint256& hash, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); - return FillBlock(chainman().m_blockman.LookupBlockIndex(hash), block, lock, chainman().ActiveChain()); + return FillBlock(chainman().m_blockman.LookupBlockIndex(hash), block, lock, chainman().ActiveChain(), chainman().m_blockman); } bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); const CChain& active = chainman().ActiveChain(); - return FillBlock(active.FindEarliestAtLeast(min_time, min_height), block, lock, active); + return FillBlock(active.FindEarliestAtLeast(min_time, min_height), block, lock, active, chainman().m_blockman); } bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out) override { @@ -571,10 +575,10 @@ public: const CChain& active = chainman().ActiveChain(); if (const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { - return FillBlock(ancestor, ancestor_out, lock, active); + return FillBlock(ancestor, ancestor_out, lock, active, chainman().m_blockman); } } - return FillBlock(nullptr, ancestor_out, lock, active); + return FillBlock(nullptr, ancestor_out, lock, active, chainman().m_blockman); } bool findAncestorByHash(const uint256& block_hash, const uint256& ancestor_hash, const FoundBlock& ancestor_out) override { @@ -582,7 +586,7 @@ public: const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash); const CBlockIndex* ancestor = chainman().m_blockman.LookupBlockIndex(ancestor_hash); if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; - return FillBlock(ancestor, ancestor_out, lock, chainman().ActiveChain()); + return FillBlock(ancestor, ancestor_out, lock, chainman().ActiveChain(), chainman().m_blockman); } bool findCommonAncestor(const uint256& block_hash1, const uint256& block_hash2, const FoundBlock& ancestor_out, const FoundBlock& block1_out, const FoundBlock& block2_out) override { @@ -594,9 +598,9 @@ public: // Using & instead of && below to avoid short circuiting and leaving // output uninitialized. Cast bool to int to avoid -Wbitwise-instead-of-logical // compiler warnings. - return int{FillBlock(ancestor, ancestor_out, lock, active)} & - int{FillBlock(block1, block1_out, lock, active)} & - int{FillBlock(block2, block2_out, lock, active)}; + return int{FillBlock(ancestor, ancestor_out, lock, active, chainman().m_blockman)} & + int{FillBlock(block1, block1_out, lock, active, chainman().m_blockman)} & + int{FillBlock(block2, block2_out, lock, active, chainman().m_blockman)}; } void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override @@ -744,27 +748,27 @@ public: RPCRunLater(name, std::move(fn), seconds); } int rpcSerializationFlags() override { return RPCSerializationFlags(); } - util::SettingsValue getSetting(const std::string& name) override + common::SettingsValue getSetting(const std::string& name) override { return args().GetSetting(name); } - std::vector<util::SettingsValue> getSettingsList(const std::string& name) override + std::vector<common::SettingsValue> getSettingsList(const std::string& name) override { return args().GetSettingsList(name); } - util::SettingsValue getRwSetting(const std::string& name) override + common::SettingsValue getRwSetting(const std::string& name) override { - util::SettingsValue result; - args().LockSettings([&](const util::Settings& settings) { - if (const util::SettingsValue* value = util::FindKey(settings.rw_settings, name)) { + common::SettingsValue result; + args().LockSettings([&](const common::Settings& settings) { + if (const common::SettingsValue* value = common::FindKey(settings.rw_settings, name)) { result = *value; } }); return result; } - bool updateRwSetting(const std::string& name, const util::SettingsValue& value, bool write) override + bool updateRwSetting(const std::string& name, const common::SettingsValue& value, bool write) override { - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { if (value.isNull()) { settings.rw_settings.erase(name); } else { diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp new file mode 100644 index 0000000000..926b157f3b --- /dev/null +++ b/src/node/kernel_notifications.cpp @@ -0,0 +1,75 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/kernel_notifications.h> + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <common/args.h> +#include <common/system.h> +#include <node/interface_ui.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/translation.h> +#include <warnings.h> + +#include <cstdint> +#include <string> +#include <thread> + +static void AlertNotify(const std::string& strMessage) +{ + uiInterface.NotifyAlertChanged(); +#if HAVE_SYSTEM + std::string strCmd = gArgs.GetArg("-alertnotify", ""); + if (strCmd.empty()) return; + + // Alert text should be plain ascii coming from a trusted source, but to + // be safe we first strip anything not in safeChars, then add single quotes around + // the whole string before passing it to the shell: + std::string singleQuote("'"); + std::string safeStatus = SanitizeString(strMessage); + safeStatus = singleQuote+safeStatus+singleQuote; + ReplaceAll(strCmd, "%s", safeStatus); + + std::thread t(runCommand, strCmd); + t.detach(); // thread runs free +#endif +} + +static void DoWarning(const bilingual_str& warning) +{ + static bool fWarned = false; + SetMiscWarning(warning); + if (!fWarned) { + AlertNotify(warning.original); + fWarned = true; + } +} + +namespace node { + +void KernelNotifications::blockTip(SynchronizationState state, CBlockIndex& index) +{ + uiInterface.NotifyBlockTip(state, &index); +} + +void KernelNotifications::headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) +{ + uiInterface.NotifyHeaderTip(state, height, timestamp, presync); +} + +void KernelNotifications::progress(const bilingual_str& title, int progress_percent, bool resume_possible) +{ + uiInterface.ShowProgress(title.translated, progress_percent, resume_possible); +} + +void KernelNotifications::warning(const bilingual_str& warning) +{ + DoWarning(warning); +} + +} // namespace node diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h new file mode 100644 index 0000000000..3e665bbf14 --- /dev/null +++ b/src/node/kernel_notifications.h @@ -0,0 +1,31 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_KERNEL_NOTIFICATIONS_H +#define BITCOIN_NODE_KERNEL_NOTIFICATIONS_H + +#include <kernel/notifications_interface.h> + +#include <cstdint> +#include <string> + +class CBlockIndex; +enum class SynchronizationState; +struct bilingual_str; + +namespace node { +class KernelNotifications : public kernel::Notifications +{ +public: + void blockTip(SynchronizationState state, CBlockIndex& index) override; + + void headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) override; + + void progress(const bilingual_str& title, int progress_percent, bool resume_possible) override; + + void warning(const bilingual_str& warning) override; +}; +} // namespace node + +#endif // BITCOIN_NODE_KERNEL_NOTIFICATIONS_H diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index f193469506..5381902263 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -38,7 +38,7 @@ void ApplyArgsManOptions(const ArgsManager& argsman, MemPoolLimits& mempool_limi } } -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, MemPoolOptions& mempool_opts) +util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, MemPoolOptions& mempool_opts) { mempool_opts.check_ratio = argsman.GetIntArg("-checkmempool", mempool_opts.check_ratio); @@ -52,7 +52,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con if (std::optional<CAmount> inc_relay_fee = ParseMoney(argsman.GetArg("-incrementalrelayfee", ""))) { mempool_opts.incremental_relay_feerate = CFeeRate{inc_relay_fee.value()}; } else { - return AmountErrMsg("incrementalrelayfee", argsman.GetArg("-incrementalrelayfee", "")); + return util::Error{AmountErrMsg("incrementalrelayfee", argsman.GetArg("-incrementalrelayfee", ""))}; } } @@ -61,7 +61,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con // High fee check is done afterward in CWallet::Create() mempool_opts.min_relay_feerate = CFeeRate{min_relay_feerate.value()}; } else { - return AmountErrMsg("minrelaytxfee", argsman.GetArg("-minrelaytxfee", "")); + return util::Error{AmountErrMsg("minrelaytxfee", argsman.GetArg("-minrelaytxfee", ""))}; } } else if (mempool_opts.incremental_relay_feerate > mempool_opts.min_relay_feerate) { // Allow only setting incremental fee to control both @@ -75,7 +75,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con if (std::optional<CAmount> parsed = ParseMoney(argsman.GetArg("-dustrelayfee", ""))) { mempool_opts.dust_relay_feerate = CFeeRate{parsed.value()}; } else { - return AmountErrMsg("dustrelayfee", argsman.GetArg("-dustrelayfee", "")); + return util::Error{AmountErrMsg("dustrelayfee", argsman.GetArg("-dustrelayfee", ""))}; } } @@ -89,12 +89,12 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con mempool_opts.require_standard = !argsman.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (!chainparams.IsTestChain() && !mempool_opts.require_standard) { - return strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.NetworkIDString()); + return util::Error{strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString())}; } mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf); ApplyArgsManOptions(argsman, mempool_opts.limits); - return std::nullopt; + return {}; } diff --git a/src/node/mempool_args.h b/src/node/mempool_args.h index 52d8b4f265..630fee6421 100644 --- a/src/node/mempool_args.h +++ b/src/node/mempool_args.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_NODE_MEMPOOL_ARGS_H #define BITCOIN_NODE_MEMPOOL_ARGS_H -#include <optional> +#include <util/result.h> class ArgsManager; class CChainParams; @@ -21,7 +21,7 @@ struct MemPoolOptions; * @param[in] argsman The ArgsManager in which to check set options. * @param[in,out] mempool_opts The MemPoolOptions to modify according to \p argsman. */ -[[nodiscard]] std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, kernel::MemPoolOptions& mempool_opts); +[[nodiscard]] util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, kernel::MemPoolOptions& mempool_opts); #endif // BITCOIN_NODE_MEMPOOL_ARGS_H diff --git a/src/node/miner.h b/src/node/miner.h index f1ccffff55..70de9e1db0 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -14,7 +14,10 @@ #include <optional> #include <stdint.h> +#include <boost/multi_index/identity.hpp> +#include <boost/multi_index/indexed_by.hpp> #include <boost/multi_index/ordered_index.hpp> +#include <boost/multi_index/tag.hpp> #include <boost/multi_index_container.hpp> class ArgsManager; diff --git a/src/node/mini_miner.cpp b/src/node/mini_miner.cpp new file mode 100644 index 0000000000..6f253eddfa --- /dev/null +++ b/src/node/mini_miner.cpp @@ -0,0 +1,371 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/mini_miner.h> + +#include <consensus/amount.h> +#include <policy/feerate.h> +#include <primitives/transaction.h> +#include <timedata.h> +#include <util/check.h> +#include <util/moneystr.h> + +#include <algorithm> +#include <numeric> +#include <utility> + +namespace node { + +MiniMiner::MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& outpoints) +{ + LOCK(mempool.cs); + // Find which outpoints to calculate bump fees for. + // Anything that's spent by the mempool is to-be-replaced + // Anything otherwise unavailable just has a bump fee of 0 + for (const auto& outpoint : outpoints) { + if (!mempool.exists(GenTxid::Txid(outpoint.hash))) { + // This UTXO is either confirmed or not yet submitted to mempool. + // If it's confirmed, no bump fee is required. + // If it's not yet submitted, we have no information, so return 0. + m_bump_fees.emplace(outpoint, 0); + continue; + } + + // UXTO is created by transaction in mempool, add to map. + // Note: This will either create a missing entry or add the outpoint to an existing entry + m_requested_outpoints_by_txid[outpoint.hash].push_back(outpoint); + + if (const auto ptx{mempool.GetConflictTx(outpoint)}) { + // This outpoint is already being spent by another transaction in the mempool. We + // assume that the caller wants to replace this transaction and its descendants. It + // would be unusual for the transaction to have descendants as the wallet won’t normally + // attempt to replace transactions with descendants. If the outpoint is from a mempool + // transaction, we still need to calculate its ancestors bump fees (added to + // m_requested_outpoints_by_txid below), but after removing the to-be-replaced entries. + // + // Note that the descendants of a transaction include the transaction itself. Also note, + // that this is only calculating bump fees. RBF fee rules should be handled separately. + CTxMemPool::setEntries descendants; + mempool.CalculateDescendants(mempool.GetIter(ptx->GetHash()).value(), descendants); + for (const auto& desc_txiter : descendants) { + m_to_be_replaced.insert(desc_txiter->GetTx().GetHash()); + } + } + } + + // No unconfirmed UTXOs, so nothing mempool-related needs to be calculated. + if (m_requested_outpoints_by_txid.empty()) return; + + // Calculate the cluster and construct the entry map. + std::vector<uint256> txids_needed; + txids_needed.reserve(m_requested_outpoints_by_txid.size()); + for (const auto& [txid, _]: m_requested_outpoints_by_txid) { + txids_needed.push_back(txid); + } + const auto cluster = mempool.GatherClusters(txids_needed); + if (cluster.empty()) { + // An empty cluster means that at least one of the transactions is missing from the mempool + // (should not be possible given processing above) or DoS limit was hit. + m_ready_to_calculate = false; + return; + } + + // Add every entry to m_entries_by_txid and m_entries, except the ones that will be replaced. + for (const auto& txiter : cluster) { + if (!m_to_be_replaced.count(txiter->GetTx().GetHash())) { + auto [mapiter, success] = m_entries_by_txid.emplace(txiter->GetTx().GetHash(), MiniMinerMempoolEntry(txiter)); + m_entries.push_back(mapiter); + } else { + auto outpoints_it = m_requested_outpoints_by_txid.find(txiter->GetTx().GetHash()); + if (outpoints_it != m_requested_outpoints_by_txid.end()) { + // This UTXO is the output of a to-be-replaced transaction. Bump fee is 0; spending + // this UTXO is impossible as it will no longer exist after the replacement. + for (const auto& outpoint : outpoints_it->second) { + m_bump_fees.emplace(outpoint, 0); + } + m_requested_outpoints_by_txid.erase(outpoints_it); + } + } + } + + // Build the m_descendant_set_by_txid cache. + for (const auto& txiter : cluster) { + const auto& txid = txiter->GetTx().GetHash(); + // Cache descendants for future use. Unlike the real mempool, a descendant MiniMinerMempoolEntry + // will not exist without its ancestor MiniMinerMempoolEntry, so these sets won't be invalidated. + std::vector<MockEntryMap::iterator> cached_descendants; + const bool remove{m_to_be_replaced.count(txid) > 0}; + CTxMemPool::setEntries descendants; + mempool.CalculateDescendants(txiter, descendants); + Assume(descendants.count(txiter) > 0); + for (const auto& desc_txiter : descendants) { + const auto txid_desc = desc_txiter->GetTx().GetHash(); + const bool remove_desc{m_to_be_replaced.count(txid_desc) > 0}; + auto desc_it{m_entries_by_txid.find(txid_desc)}; + Assume((desc_it == m_entries_by_txid.end()) == remove_desc); + if (remove) Assume(remove_desc); + // It's possible that remove=false but remove_desc=true. + if (!remove && !remove_desc) { + cached_descendants.push_back(desc_it); + } + } + if (remove) { + Assume(cached_descendants.empty()); + } else { + m_descendant_set_by_txid.emplace(txid, cached_descendants); + } + } + + // Release the mempool lock; we now have all the information we need for a subset of the entries + // we care about. We will solely operate on the MiniMinerMempoolEntry map from now on. + Assume(m_in_block.empty()); + Assume(m_requested_outpoints_by_txid.size() <= outpoints.size()); + SanityCheck(); +} + +// Compare by min(ancestor feerate, individual feerate), then iterator +// +// Under the ancestor-based mining approach, high-feerate children can pay for parents, but high-feerate +// parents do not incentive inclusion of their children. Therefore the mining algorithm only considers +// transactions for inclusion on basis of the minimum of their own feerate or their ancestor feerate. +struct AncestorFeerateComparator +{ + template<typename I> + bool operator()(const I& a, const I& b) const { + auto min_feerate = [](const MiniMinerMempoolEntry& e) -> CFeeRate { + const CAmount ancestor_fee{e.GetModFeesWithAncestors()}; + const int64_t ancestor_size{e.GetSizeWithAncestors()}; + const CAmount tx_fee{e.GetModifiedFee()}; + const int64_t tx_size{e.GetTxSize()}; + // Comparing ancestor feerate with individual feerate: + // ancestor_fee / ancestor_size <= tx_fee / tx_size + // Avoid division and possible loss of precision by + // multiplying both sides by the sizes: + return ancestor_fee * tx_size < tx_fee * ancestor_size ? + CFeeRate(ancestor_fee, ancestor_size) : + CFeeRate(tx_fee, tx_size); + }; + CFeeRate a_feerate{min_feerate(a->second)}; + CFeeRate b_feerate{min_feerate(b->second)}; + if (a_feerate != b_feerate) { + return a_feerate > b_feerate; + } + // Use txid as tiebreaker for stable sorting + return a->first < b->first; + } +}; + +void MiniMiner::DeleteAncestorPackage(const std::set<MockEntryMap::iterator, IteratorComparator>& ancestors) +{ + Assume(ancestors.size() >= 1); + // "Mine" all transactions in this ancestor set. + for (auto& anc : ancestors) { + Assume(m_in_block.count(anc->first) == 0); + m_in_block.insert(anc->first); + m_total_fees += anc->second.GetModifiedFee(); + m_total_vsize += anc->second.GetTxSize(); + auto it = m_descendant_set_by_txid.find(anc->first); + // Each entry’s descendant set includes itself + Assume(it != m_descendant_set_by_txid.end()); + for (auto& descendant : it->second) { + // If these fail, we must be double-deducting. + Assume(descendant->second.GetModFeesWithAncestors() >= anc->second.GetModifiedFee()); + Assume(descendant->second.vsize_with_ancestors >= anc->second.GetTxSize()); + descendant->second.fee_with_ancestors -= anc->second.GetModifiedFee(); + descendant->second.vsize_with_ancestors -= anc->second.GetTxSize(); + } + } + // Delete these entries. + for (const auto& anc : ancestors) { + m_descendant_set_by_txid.erase(anc->first); + // The above loop should have deducted each ancestor's size and fees from each of their + // respective descendants exactly once. + Assume(anc->second.GetModFeesWithAncestors() == 0); + Assume(anc->second.GetSizeWithAncestors() == 0); + auto vec_it = std::find(m_entries.begin(), m_entries.end(), anc); + Assume(vec_it != m_entries.end()); + m_entries.erase(vec_it); + m_entries_by_txid.erase(anc); + } +} + +void MiniMiner::SanityCheck() const +{ + // m_entries, m_entries_by_txid, and m_descendant_set_by_txid all same size + Assume(m_entries.size() == m_entries_by_txid.size()); + Assume(m_entries.size() == m_descendant_set_by_txid.size()); + // Cached ancestor values should be at least as large as the transaction's own fee and size + Assume(std::all_of(m_entries.begin(), m_entries.end(), [](const auto& entry) { + return entry->second.GetSizeWithAncestors() >= entry->second.GetTxSize() && + entry->second.GetModFeesWithAncestors() >= entry->second.GetModifiedFee();})); + // None of the entries should be to-be-replaced transactions + Assume(std::all_of(m_to_be_replaced.begin(), m_to_be_replaced.end(), + [&](const auto& txid){return m_entries_by_txid.find(txid) == m_entries_by_txid.end();})); +} + +void MiniMiner::BuildMockTemplate(const CFeeRate& target_feerate) +{ + while (!m_entries_by_txid.empty()) { + // Sort again, since transaction removal may change some m_entries' ancestor feerates. + std::sort(m_entries.begin(), m_entries.end(), AncestorFeerateComparator()); + + // Pick highest ancestor feerate entry. + auto best_iter = m_entries.begin(); + Assume(best_iter != m_entries.end()); + const auto ancestor_package_size = (*best_iter)->second.GetSizeWithAncestors(); + const auto ancestor_package_fee = (*best_iter)->second.GetModFeesWithAncestors(); + // Stop here. Everything that didn't "make it into the block" has bumpfee. + if (ancestor_package_fee < target_feerate.GetFee(ancestor_package_size)) { + break; + } + + // Calculate ancestors on the fly. This lookup should be fairly cheap, and ancestor sets + // change at every iteration, so this is more efficient than maintaining a cache. + std::set<MockEntryMap::iterator, IteratorComparator> ancestors; + { + std::set<MockEntryMap::iterator, IteratorComparator> to_process; + to_process.insert(*best_iter); + while (!to_process.empty()) { + auto iter = to_process.begin(); + Assume(iter != to_process.end()); + ancestors.insert(*iter); + for (const auto& input : (*iter)->second.GetTx().vin) { + if (auto parent_it{m_entries_by_txid.find(input.prevout.hash)}; parent_it != m_entries_by_txid.end()) { + if (ancestors.count(parent_it) == 0) { + to_process.insert(parent_it); + } + } + } + to_process.erase(iter); + } + } + DeleteAncestorPackage(ancestors); + SanityCheck(); + } + Assume(m_in_block.empty() || m_total_fees >= target_feerate.GetFee(m_total_vsize)); + // Do not try to continue building the block template with a different feerate. + m_ready_to_calculate = false; +} + +std::map<COutPoint, CAmount> MiniMiner::CalculateBumpFees(const CFeeRate& target_feerate) +{ + if (!m_ready_to_calculate) return {}; + // Build a block template until the target feerate is hit. + BuildMockTemplate(target_feerate); + + // Each transaction that "made it into the block" has a bumpfee of 0, i.e. they are part of an + // ancestor package with at least the target feerate and don't need to be bumped. + for (const auto& txid : m_in_block) { + // Not all of the block transactions were necessarily requested. + auto it = m_requested_outpoints_by_txid.find(txid); + if (it != m_requested_outpoints_by_txid.end()) { + for (const auto& outpoint : it->second) { + m_bump_fees.emplace(outpoint, 0); + } + m_requested_outpoints_by_txid.erase(it); + } + } + + // A transactions and its ancestors will only be picked into a block when + // both the ancestor set feerate and the individual feerate meet the target + // feerate. + // + // We had to convince ourselves that after running the mini miner and + // picking all eligible transactions into our MockBlockTemplate, there + // could still be transactions remaining that have a lower individual + // feerate than their ancestor feerate. So here is an example: + // + // ┌─────────────────┐ + // │ │ + // │ Grandparent │ + // │ 1700 vB │ + // │ 1700 sats │ Target feerate: 10 s/vB + // │ 1 s/vB │ GP Ancestor Set Feerate (ASFR): 1 s/vB + // │ │ P1_ASFR: 9.84 s/vB + // └──────▲───▲──────┘ P2_ASFR: 2.47 s/vB + // │ │ C_ASFR: 10.27 s/vB + // ┌───────────────┐ │ │ ┌──────────────┐ + // │ ├────┘ └────┤ │ ⇒ C_FR < TFR < C_ASFR + // │ Parent 1 │ │ Parent 2 │ + // │ 200 vB │ │ 200 vB │ + // │ 17000 sats │ │ 3000 sats │ + // │ 85 s/vB │ │ 15 s/vB │ + // │ │ │ │ + // └───────────▲───┘ └───▲──────────┘ + // │ │ + // │ ┌───────────┐ │ + // └────┤ ├────┘ + // │ Child │ + // │ 100 vB │ + // │ 900 sats │ + // │ 9 s/vB │ + // │ │ + // └───────────┘ + // + // We therefore calculate both the bump fee that is necessary to elevate + // the individual transaction to the target feerate: + // target_feerate × tx_size - tx_fees + // and the bump fee that is necessary to bump the entire ancestor set to + // the target feerate: + // target_feerate × ancestor_set_size - ancestor_set_fees + // By picking the maximum from the two, we ensure that a transaction meets + // both criteria. + for (const auto& [txid, outpoints] : m_requested_outpoints_by_txid) { + auto it = m_entries_by_txid.find(txid); + Assume(it != m_entries_by_txid.end()); + if (it != m_entries_by_txid.end()) { + Assume(target_feerate.GetFee(it->second.GetSizeWithAncestors()) > std::min(it->second.GetModifiedFee(), it->second.GetModFeesWithAncestors())); + CAmount bump_fee_with_ancestors = target_feerate.GetFee(it->second.GetSizeWithAncestors()) - it->second.GetModFeesWithAncestors(); + CAmount bump_fee_individual = target_feerate.GetFee(it->second.GetTxSize()) - it->second.GetModifiedFee(); + const CAmount bump_fee{std::max(bump_fee_with_ancestors, bump_fee_individual)}; + Assume(bump_fee >= 0); + for (const auto& outpoint : outpoints) { + m_bump_fees.emplace(outpoint, bump_fee); + } + } + } + return m_bump_fees; +} + +std::optional<CAmount> MiniMiner::CalculateTotalBumpFees(const CFeeRate& target_feerate) +{ + if (!m_ready_to_calculate) return std::nullopt; + // Build a block template until the target feerate is hit. + BuildMockTemplate(target_feerate); + + // All remaining ancestors that are not part of m_in_block must be bumped, but no other relatives + std::set<MockEntryMap::iterator, IteratorComparator> ancestors; + std::set<MockEntryMap::iterator, IteratorComparator> to_process; + for (const auto& [txid, outpoints] : m_requested_outpoints_by_txid) { + // Skip any ancestors that already have a miner score higher than the target feerate + // (already "made it" into the block) + if (m_in_block.count(txid)) continue; + auto iter = m_entries_by_txid.find(txid); + if (iter == m_entries_by_txid.end()) continue; + to_process.insert(iter); + ancestors.insert(iter); + } + + std::set<uint256> has_been_processed; + while (!to_process.empty()) { + auto iter = to_process.begin(); + const CTransaction& tx = (*iter)->second.GetTx(); + for (const auto& input : tx.vin) { + if (auto parent_it{m_entries_by_txid.find(input.prevout.hash)}; parent_it != m_entries_by_txid.end()) { + if (!has_been_processed.count(input.prevout.hash)) { + to_process.insert(parent_it); + } + ancestors.insert(parent_it); + } + } + has_been_processed.insert(tx.GetHash()); + to_process.erase(iter); + } + const auto ancestor_package_size = std::accumulate(ancestors.cbegin(), ancestors.cend(), int64_t{0}, + [](int64_t sum, const auto it) {return sum + it->second.GetTxSize();}); + const auto ancestor_package_fee = std::accumulate(ancestors.cbegin(), ancestors.cend(), CAmount{0}, + [](CAmount sum, const auto it) {return sum + it->second.GetModifiedFee();}); + return target_feerate.GetFee(ancestor_package_size) - ancestor_package_fee; +} +} // namespace node diff --git a/src/node/mini_miner.h b/src/node/mini_miner.h new file mode 100644 index 0000000000..db07e6d1bf --- /dev/null +++ b/src/node/mini_miner.h @@ -0,0 +1,121 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_MINI_MINER_H +#define BITCOIN_NODE_MINI_MINER_H + +#include <txmempool.h> + +#include <memory> +#include <optional> +#include <stdint.h> + +namespace node { + +// Container for tracking updates to ancestor feerate as we include ancestors in the "block" +class MiniMinerMempoolEntry +{ + const CAmount fee_individual; + const CTransactionRef tx; + const int64_t vsize_individual; + +// This class must be constructed while holding mempool.cs. After construction, the object's +// methods can be called without holding that lock. +public: + CAmount fee_with_ancestors; + int64_t vsize_with_ancestors; + explicit MiniMinerMempoolEntry(CTxMemPool::txiter entry) : + fee_individual{entry->GetModifiedFee()}, + tx{entry->GetSharedTx()}, + vsize_individual(entry->GetTxSize()), + fee_with_ancestors{entry->GetModFeesWithAncestors()}, + vsize_with_ancestors(entry->GetSizeWithAncestors()) + { } + + CAmount GetModifiedFee() const { return fee_individual; } + CAmount GetModFeesWithAncestors() const { return fee_with_ancestors; } + int64_t GetTxSize() const { return vsize_individual; } + int64_t GetSizeWithAncestors() const { return vsize_with_ancestors; } + const CTransaction& GetTx() const LIFETIMEBOUND { return *tx; } +}; + +// Comparator needed for std::set<MockEntryMap::iterator> +struct IteratorComparator +{ + template<typename I> + bool operator()(const I& a, const I& b) const + { + return &(*a) < &(*b); + } +}; + +/** A minimal version of BlockAssembler. Allows us to run the mining algorithm on a subset of + * mempool transactions, ignoring consensus rules, to calculate mining scores. */ +class MiniMiner +{ + // When true, a caller may use CalculateBumpFees(). Becomes false if we failed to retrieve + // mempool entries (i.e. cluster size too large) or bump fees have already been calculated. + bool m_ready_to_calculate{true}; + + // Set once per lifetime, fill in during initialization. + // txids of to-be-replaced transactions + std::set<uint256> m_to_be_replaced; + + // If multiple argument outpoints correspond to the same transaction, cache them together in + // a single entry indexed by txid. Then we can just work with txids since all outpoints from + // the same tx will have the same bumpfee. Excludes non-mempool transactions. + std::map<uint256, std::vector<COutPoint>> m_requested_outpoints_by_txid; + + // What we're trying to calculate. + std::map<COutPoint, CAmount> m_bump_fees; + + // The constructed block template + std::set<uint256> m_in_block; + + // Information on the current status of the block + CAmount m_total_fees{0}; + int32_t m_total_vsize{0}; + + /** Main data structure holding the entries, can be indexed by txid */ + std::map<uint256, MiniMinerMempoolEntry> m_entries_by_txid; + using MockEntryMap = decltype(m_entries_by_txid); + + /** Vector of entries, can be sorted by ancestor feerate. */ + std::vector<MockEntryMap::iterator> m_entries; + + /** Map of txid to its descendants. Should be inclusive. */ + std::map<uint256, std::vector<MockEntryMap::iterator>> m_descendant_set_by_txid; + + /** Consider this ancestor package "mined" so remove all these entries from our data structures. */ + void DeleteAncestorPackage(const std::set<MockEntryMap::iterator, IteratorComparator>& ancestors); + + /** Perform some checks. */ + void SanityCheck() const; + +public: + /** Returns true if CalculateBumpFees may be called, false if not. */ + bool IsReadyToCalculate() const { return m_ready_to_calculate; } + + /** Build a block template until the target feerate is hit. */ + void BuildMockTemplate(const CFeeRate& target_feerate); + + /** Returns set of txids in the block template if one has been constructed. */ + std::set<uint256> GetMockTemplateTxids() const { return m_in_block; } + + MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& outpoints); + + /** Construct a new block template and, for each outpoint corresponding to a transaction that + * did not make it into the block, calculate the cost of bumping those transactions (and their + * ancestors) to the minimum feerate. Returns a map from outpoint to bump fee, or an empty map + * if they cannot be calculated. */ + std::map<COutPoint, CAmount> CalculateBumpFees(const CFeeRate& target_feerate); + + /** Construct a new block template and, calculate the cost of bumping all transactions that did + * not make it into the block to the target feerate. Returns the total bump fee, or std::nullopt + * if it cannot be calculated. */ + std::optional<CAmount> CalculateTotalBumpFees(const CFeeRate& target_feerate); +}; +} // namespace node + +#endif // BITCOIN_NODE_MINI_MINER_H diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index c7c8493f0c..026c8084dd 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -122,7 +122,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t return TransactionError::OK; } -CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock) +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, uint256& hashBlock, const BlockManager& blockman) { if (mempool && !block_index) { CTransactionRef ptx = mempool->get(hash); @@ -143,7 +143,7 @@ CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMe } if (block_index) { CBlock block; - if (ReadBlockFromDisk(block, block_index, consensusParams)) { + if (blockman.ReadBlockFromDisk(block, *block_index)) { for (const auto& tx : block.vtx) { if (tx->GetHash() == hash) { hashBlock = block_index->GetBlockHash(); diff --git a/src/node/transaction.h b/src/node/transaction.h index 45f174f13c..168273594c 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -16,6 +16,7 @@ struct Params; } namespace node { +class BlockManager; struct NodeContext; /** Maximum fee rate for sendrawtransaction and testmempoolaccept RPC calls. @@ -53,11 +54,10 @@ static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; * @param[in] block_index The block to read from disk, or nullptr * @param[in] mempool If provided, check mempool for tx * @param[in] hash The txid - * @param[in] consensusParams The params * @param[out] hashBlock The block hash, if the tx was found via -txindex or block_index * @returns The tx if found, otherwise nullptr */ -CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, uint256& hashBlock, const BlockManager& blockman); } // namespace node #endif // BITCOIN_NODE_TRANSACTION_H diff --git a/src/node/txreconciliation.cpp b/src/node/txreconciliation.cpp index 9938759074..d62046daaa 100644 --- a/src/node/txreconciliation.cpp +++ b/src/node/txreconciliation.cpp @@ -4,9 +4,9 @@ #include <node/txreconciliation.h> +#include <common/system.h> #include <logging.h> #include <util/check.h> -#include <util/system.h> #include <unordered_map> #include <variant> diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp index 3dae46fb84..036a25d0a5 100644 --- a/src/node/utxo_snapshot.cpp +++ b/src/node/utxo_snapshot.cpp @@ -4,7 +4,6 @@ #include <node/utxo_snapshot.h> -#include <common/args.h> #include <logging.h> #include <streams.h> #include <sync.h> @@ -82,10 +81,10 @@ std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) return base_blockhash; } -std::optional<fs::path> FindSnapshotChainstateDir() +std::optional<fs::path> FindSnapshotChainstateDir(const fs::path& data_dir) { fs::path possible_dir = - gArgs.GetDataDirNet() / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX)); + data_dir / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX)); if (fs::exists(possible_dir)) { return possible_dir; diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 44ddd77dc3..a6dd3f3f13 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -67,7 +67,7 @@ constexpr std::string_view SNAPSHOT_CHAINSTATE_SUFFIX = "_snapshot"; //! Return a path to the snapshot-based chainstate dir, if one exists. -std::optional<fs::path> FindSnapshotChainstateDir(); +std::optional<fs::path> FindSnapshotChainstateDir(const fs::path& data_dir); } // namespace node diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 6121224979..ae226f7011 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -6,6 +6,7 @@ #include <policy/fees.h> #include <clientversion.h> +#include <common/system.h> #include <consensus/amount.h> #include <kernel/mempool_entry.h> #include <logging.h> @@ -19,7 +20,6 @@ #include <uint256.h> #include <util/fs.h> #include <util/serfloat.h> -#include <util/system.h> #include <util/time.h> #include <algorithm> diff --git a/src/policy/policy.h b/src/policy/policy.h index 394fb34230..9135cae91c 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -24,7 +24,7 @@ static constexpr unsigned int DEFAULT_BLOCK_MAX_WEIGHT{MAX_BLOCK_WEIGHT - 4000}; /** Default for -blockmintxfee, which sets the minimum feerate for a transaction in blocks created by mining code **/ static constexpr unsigned int DEFAULT_BLOCK_MIN_TX_FEE{1000}; /** The maximum weight for transactions we're willing to relay/mine */ -static constexpr unsigned int MAX_STANDARD_TX_WEIGHT{400000}; +static constexpr int32_t MAX_STANDARD_TX_WEIGHT{400000}; /** The minimum non-witness size for transactions we're willing to relay/mine: one larger than 64 */ static constexpr unsigned int MIN_STANDARD_TX_NONWITNESS_SIZE{65}; /** Maximum number of signature check operations in an IsStandard() P2SH script */ diff --git a/src/protocol.cpp b/src/protocol.cpp index 5725813b7e..5ecaabec36 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -5,7 +5,7 @@ #include <protocol.h> -#include <util/system.h> +#include <common/system.h> #include <atomic> diff --git a/src/protocol.h b/src/protocol.h index cbcd400fef..ac4545c311 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -134,7 +134,8 @@ extern const char* GETADDR; /** * The mempool message requests the TXIDs of transactions that the receiving * node has verified as valid but which have not yet appeared in a block. - * @since protocol version 60002. + * @since protocol version 60002 as described by BIP35. + * Only available with service bit NODE_BLOOM, see also BIP111. */ extern const char* MEMPOOL; /** @@ -278,8 +279,6 @@ enum ServiceFlags : uint64_t { // set by all Bitcoin Core non pruned nodes, and is unset by SPV clients or other light clients. NODE_NETWORK = (1 << 0), // NODE_BLOOM means the node is capable and willing to handle bloom-filtered connections. - // Bitcoin Core nodes used to support this by default, without advertising this bit, - // but no longer do as of protocol version 70011 (= NO_BLOOM_VERSION) NODE_BLOOM = (1 << 2), // NODE_WITNESS indicates that a node can be asked for blocks and transactions including // witness data. diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 0d0f1a4d15..e4689e4389 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -53,15 +53,14 @@ struct AddressTableEntryLessThan }; /* Determine address type from address purpose */ -static AddressTableEntry::Type translateTransactionType(wallet::AddressPurpose purpose, bool isMine) +constexpr AddressTableEntry::Type translateTransactionType(wallet::AddressPurpose purpose, bool isMine) { // "refund" addresses aren't shown, and change addresses aren't returned by getAddresses at all. switch (purpose) { case wallet::AddressPurpose::SEND: return AddressTableEntry::Sending; case wallet::AddressPurpose::RECEIVE: return AddressTableEntry::Receiving; case wallet::AddressPurpose::REFUND: return AddressTableEntry::Hidden; - // No default case to allow for compiler to warn - } + } // no default case, so the compiler can warn about missing cases assert(false); } diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 2f00009596..8f45af9485 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -9,8 +9,10 @@ #include <qt/bitcoin.h> #include <chainparams.h> +#include <node/context.h> #include <common/args.h> #include <common/init.h> +#include <common/system.h> #include <init.h> #include <interfaces/handler.h> #include <interfaces/init.h> @@ -32,7 +34,6 @@ #include <uint256.h> #include <util/exception.h> #include <util/string.h> -#include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> #include <validation.h> @@ -397,9 +398,7 @@ void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHead { qDebug() << __func__ << ": Initialization result: " << success; - // Set exit result. - returnValue = success ? EXIT_SUCCESS : EXIT_FAILURE; - if(success) { + if (success) { delete m_splash; m_splash = nullptr; @@ -601,7 +600,7 @@ int GuiMain(int argc, char* argv[]) PaymentServer::ipcParseCommandLine(argc, argv); #endif - QScopedPointer<const NetworkStyle> networkStyle(NetworkStyle::instantiate(Params().NetworkIDString())); + QScopedPointer<const NetworkStyle> networkStyle(NetworkStyle::instantiate(Params().GetChainType())); assert(!networkStyle.isNull()); // Allow for separate UI settings for testnets QApplication::setApplicationName(networkStyle->getAppName()); @@ -653,7 +652,6 @@ int GuiMain(int argc, char* argv[]) app.InitPruneSetting(prune_MiB); } - int rv = EXIT_SUCCESS; try { app.createWindow(networkStyle.data()); @@ -666,10 +664,9 @@ int GuiMain(int argc, char* argv[]) WinShutdownMonitor::registerShutdownBlockReason(QObject::tr("%1 didn't yet exit safely…").arg(PACKAGE_NAME), (HWND)app.getMainWinId()); #endif app.exec(); - rv = app.getReturnValue(); } else { // A dialog with detailed error will have been shown by InitError() - rv = EXIT_FAILURE; + return EXIT_FAILURE; } } catch (const std::exception& e) { PrintExceptionContinue(&e, "Runaway exception"); @@ -678,5 +675,5 @@ int GuiMain(int argc, char* argv[]) PrintExceptionContinue(nullptr, "Runaway exception"); app.handleRunawayException(QString::fromStdString(app.node().getWarnings().translated)); } - return rv; + return app.node().getExitStatus(); } diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 9174e23de6..9622c9d57d 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -62,9 +62,6 @@ public: /// Request core initialization void requestInitialize(); - /// Get process return value - int getReturnValue() const { return returnValue; } - /// Get window identifier of QMainWindow (BitcoinGUI) WId getMainWinId() const; @@ -104,7 +101,6 @@ private: PaymentServer* paymentServer{nullptr}; WalletController* m_wallet_controller{nullptr}; #endif - int returnValue{0}; const PlatformStyle* platformStyle{nullptr}; std::unique_ptr<QWidget> shutdownWindow; SplashScreen* m_splash = nullptr; diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index d26ef52eb4..f201d8fa01 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -30,16 +30,17 @@ #include <qt/macdockiconhandler.h> #endif -#include <functional> #include <chain.h> #include <chainparams.h> +#include <common/system.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <node/interface_ui.h> -#include <util/system.h> #include <util/translation.h> #include <validation.h> +#include <functional> + #include <QAction> #include <QActionGroup> #include <QApplication> diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 837e5f4fed..ff7405d139 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -12,11 +12,11 @@ #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <net.h> #include <netbase.h> -#include <util/system.h> #include <util/threadnames.h> #include <util/time.h> #include <validation.h> @@ -28,8 +28,8 @@ #include <QThread> #include <QTimer> -static int64_t nLastHeaderTipUpdateNotification = 0; -static int64_t nLastBlockTipUpdateNotification = 0; +static SteadyClock::time_point g_last_header_tip_update_notification{}; +static SteadyClock::time_point g_last_block_tip_update_notification{}; ClientModel::ClientModel(interfaces::Node& node, OptionsModel *_optionsModel, QObject *parent) : QObject(parent), @@ -222,9 +222,9 @@ void ClientModel::TipChanged(SynchronizationState sync_state, interfaces::BlockT // Throttle GUI notifications about (a) blocks during initial sync, and (b) both blocks and headers during reindex. const bool throttle = (sync_state != SynchronizationState::POST_INIT && synctype == SyncType::BLOCK_SYNC) || sync_state == SynchronizationState::INIT_REINDEX; - const int64_t now = throttle ? GetTimeMillis() : 0; - int64_t& nLastUpdateNotification = synctype != SyncType::BLOCK_SYNC ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; - if (throttle && now < nLastUpdateNotification + count_milliseconds(MODEL_UPDATE_DELAY)) { + const auto now{throttle ? SteadyClock::now() : SteadyClock::time_point{}}; + auto& nLastUpdateNotification = synctype != SyncType::BLOCK_SYNC ? g_last_header_tip_update_notification : g_last_block_tip_update_notification; + if (throttle && now < nLastUpdateNotification + MODEL_UPDATE_DELAY) { return; } diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index e1b1ae12e9..50d3c2e559 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -412,8 +412,7 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * unsigned int nQuantity = 0; bool fWitness = false; - std::vector<COutPoint> vCoinControl; - m_coin_control.ListSelected(vCoinControl); + auto vCoinControl{m_coin_control.ListSelected()}; size_t i = 0; for (const auto& out : model->wallet().getCoins(vCoinControl)) { diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index dc7daed4bc..8d8328aad8 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -21,6 +21,7 @@ #include <protocol.h> #include <script/script.h> #include <script/standard.h> +#include <util/chaintype.h> #include <util/exception.h> #include <util/fs.h> #include <util/fs_helpers.h> @@ -503,12 +504,12 @@ bool LabelOutOfFocusEventFilter::eventFilter(QObject* watched, QEvent* event) #ifdef WIN32 fs::path static StartupShortcutPath() { - std::string chain = gArgs.GetChainName(); - if (chain == CBaseChainParams::MAIN) + ChainType chain = gArgs.GetChainType(); + if (chain == ChainType::MAIN) return GetSpecialFolderPath(CSIDL_STARTUP) / "Bitcoin.lnk"; - if (chain == CBaseChainParams::TESTNET) // Remove this special case when CBaseChainParams::TESTNET = "testnet4" + if (chain == ChainType::TESTNET) // Remove this special case when testnet CBaseChainParams::DataDir() is incremented to "testnet4" return GetSpecialFolderPath(CSIDL_STARTUP) / "Bitcoin (testnet).lnk"; - return GetSpecialFolderPath(CSIDL_STARTUP) / fs::u8path(strprintf("Bitcoin (%s).lnk", chain)); + return GetSpecialFolderPath(CSIDL_STARTUP) / fs::u8path(strprintf("Bitcoin (%s).lnk", ChainTypeToString(chain))); } bool GetStartOnSystemStartup() @@ -541,7 +542,7 @@ bool SetStartOnSystemStartup(bool fAutoStart) // Start client minimized QString strArgs = "-min"; // Set -testnet /-regtest options - strArgs += QString::fromStdString(strprintf(" -chain=%s", gArgs.GetChainName())); + strArgs += QString::fromStdString(strprintf(" -chain=%s", gArgs.GetChainTypeString())); // Set the path to the shortcut target psl->SetPath(pszExePath); @@ -586,10 +587,10 @@ fs::path static GetAutostartDir() fs::path static GetAutostartFilePath() { - std::string chain = gArgs.GetChainName(); - if (chain == CBaseChainParams::MAIN) + ChainType chain = gArgs.GetChainType(); + if (chain == ChainType::MAIN) return GetAutostartDir() / "bitcoin.desktop"; - return GetAutostartDir() / fs::u8path(strprintf("bitcoin-%s.desktop", chain)); + return GetAutostartDir() / fs::u8path(strprintf("bitcoin-%s.desktop", ChainTypeToString(chain))); } bool GetStartOnSystemStartup() @@ -629,15 +630,15 @@ bool SetStartOnSystemStartup(bool fAutoStart) std::ofstream optionFile{GetAutostartFilePath(), std::ios_base::out | std::ios_base::trunc}; if (!optionFile.good()) return false; - std::string chain = gArgs.GetChainName(); + ChainType chain = gArgs.GetChainType(); // Write a bitcoin.desktop file to the autostart directory: optionFile << "[Desktop Entry]\n"; optionFile << "Type=Application\n"; - if (chain == CBaseChainParams::MAIN) + if (chain == ChainType::MAIN) optionFile << "Name=Bitcoin\n"; else - optionFile << strprintf("Name=Bitcoin (%s)\n", chain); - optionFile << "Exec=" << pszExePath << strprintf(" -min -chain=%s\n", chain); + optionFile << strprintf("Name=Bitcoin (%s)\n", ChainTypeToString(chain)); + optionFile << "Exec=" << pszExePath << strprintf(" -min -chain=%s\n", ChainTypeToString(chain)); optionFile << "Terminal=false\n"; optionFile << "Hidden=false\n"; optionFile.close(); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index a54ba19354..f86b167076 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -9,6 +9,7 @@ #include <chainparams.h> #include <qt/intro.h> #include <qt/forms/ui_intro.h> +#include <util/chaintype.h> #include <util/fs.h> #include <qt/guiconstants.h> @@ -219,7 +220,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) { /* Use selectParams here to guarantee Params() can be used by node interface */ try { - SelectParams(gArgs.GetChainName()); + SelectParams(gArgs.GetChainType()); } catch (const std::exception&) { return false; } diff --git a/src/qt/networkstyle.cpp b/src/qt/networkstyle.cpp index b789e6a958..b6314f5533 100644 --- a/src/qt/networkstyle.cpp +++ b/src/qt/networkstyle.cpp @@ -6,21 +6,21 @@ #include <qt/guiconstants.h> -#include <chainparamsbase.h> #include <tinyformat.h> +#include <util/chaintype.h> #include <QApplication> static const struct { - const char *networkId; + const ChainType networkId; const char *appName; const int iconColorHueShift; const int iconColorSaturationReduction; } network_styles[] = { - {"main", QAPP_APP_NAME_DEFAULT, 0, 0}, - {"test", QAPP_APP_NAME_TESTNET, 70, 30}, - {"signet", QAPP_APP_NAME_SIGNET, 35, 15}, - {"regtest", QAPP_APP_NAME_REGTEST, 160, 30}, + {ChainType::MAIN, QAPP_APP_NAME_DEFAULT, 0, 0}, + {ChainType::TESTNET, QAPP_APP_NAME_TESTNET, 70, 30}, + {ChainType::SIGNET, QAPP_APP_NAME_SIGNET, 35, 15}, + {ChainType::REGTEST, QAPP_APP_NAME_REGTEST, 160, 30}, }; // titleAddText needs to be const char* for tr() @@ -77,9 +77,9 @@ NetworkStyle::NetworkStyle(const QString &_appName, const int iconColorHueShift, trayAndWindowIcon = QIcon(pixmap.scaled(QSize(256,256))); } -const NetworkStyle* NetworkStyle::instantiate(const std::string& networkId) +const NetworkStyle* NetworkStyle::instantiate(const ChainType networkId) { - std::string titleAddText = networkId == CBaseChainParams::MAIN ? "" : strprintf("[%s]", networkId); + std::string titleAddText = networkId == ChainType::MAIN ? "" : strprintf("[%s]", ChainTypeToString(networkId)); for (const auto& network_style : network_styles) { if (networkId == network_style.networkId) { return new NetworkStyle( diff --git a/src/qt/networkstyle.h b/src/qt/networkstyle.h index a73e3e2625..dd2aee3eb3 100644 --- a/src/qt/networkstyle.h +++ b/src/qt/networkstyle.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_QT_NETWORKSTYLE_H #define BITCOIN_QT_NETWORKSTYLE_H +#include <util/chaintype.h> + #include <QIcon> #include <QPixmap> #include <QString> @@ -14,7 +16,7 @@ class NetworkStyle { public: /** Get style associated with provided network id, or 0 if not known */ - static const NetworkStyle* instantiate(const std::string& networkId); + static const NetworkStyle* instantiate(const ChainType networkId); const QString &getAppName() const { return appName; } const QIcon &getAppIcon() const { return appIcon; } diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 6dec4b2e42..512fce473d 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -15,11 +15,11 @@ #include <qt/guiutil.h> #include <qt/optionsmodel.h> +#include <common/system.h> #include <interfaces/node.h> -#include <validation.h> // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS #include <netbase.h> -#include <txdb.h> // for -dbcache defaults -#include <util/system.h> +#include <txdb.h> +#include <validation.h> #include <chrono> @@ -406,9 +406,8 @@ void OptionsDialog::updateProxyValidationState() void OptionsDialog::updateDefaultProxyNets() { - CNetAddr ui_proxy_netaddr; - LookupHost(ui->proxyIp->text().toStdString(), ui_proxy_netaddr, /*fAllowLookup=*/false); - const CService ui_proxy{ui_proxy_netaddr, ui->proxyPort->text().toUShort()}; + const std::optional<CNetAddr> ui_proxy_netaddr{LookupHost(ui->proxyIp->text().toStdString(), /*fAllowLookup=*/false)}; + const CService ui_proxy{ui_proxy_netaddr.value_or(CNetAddr{}), ui->proxyPort->text().toUShort()}; Proxy proxy; bool has_proxy; diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 87a7e703b8..c1563fe1e2 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -60,7 +60,7 @@ static const char* SettingName(OptionsModel::OptionID option) } /** Call node.updateRwSetting() with Bitcoin 22.x workaround. */ -static void UpdateRwSetting(interfaces::Node& node, OptionsModel::OptionID option, const std::string& suffix, const util::SettingsValue& value) +static void UpdateRwSetting(interfaces::Node& node, OptionsModel::OptionID option, const std::string& suffix, const common::SettingsValue& value) { if (value.isNum() && (option == OptionsModel::DatabaseCache || @@ -81,14 +81,14 @@ static void UpdateRwSetting(interfaces::Node& node, OptionsModel::OptionID optio } //! Convert enabled/size values to bitcoin -prune setting. -static util::SettingsValue PruneSetting(bool prune_enabled, int prune_size_gb) +static common::SettingsValue PruneSetting(bool prune_enabled, int prune_size_gb) { assert(!prune_enabled || prune_size_gb >= 1); // PruneSizeGB and ParsePruneSizeGB never return less return prune_enabled ? PruneGBtoMiB(prune_size_gb) : 0; } //! Get pruning enabled value to show in GUI from bitcoin -prune setting. -static bool PruneEnabled(const util::SettingsValue& prune_setting) +static bool PruneEnabled(const common::SettingsValue& prune_setting) { // -prune=1 setting is manual pruning mode, so disabled for purposes of the gui return SettingToInt(prune_setting, 0) > 1; @@ -96,7 +96,7 @@ static bool PruneEnabled(const util::SettingsValue& prune_setting) //! Get pruning size value to show in GUI from bitcoin -prune setting. If //! pruning is not enabled, just show default recommended pruning size (2GB). -static int PruneSizeGB(const util::SettingsValue& prune_setting) +static int PruneSizeGB(const common::SettingsValue& prune_setting) { int value = SettingToInt(prune_setting, 0); return value > 1 ? PruneMiBtoGB(value) : DEFAULT_PRUNE_TARGET_GB; @@ -311,8 +311,8 @@ static QString GetDefaultProxyAddress() void OptionsModel::SetPruneTargetGB(int prune_target_gb) { - const util::SettingsValue cur_value = node().getPersistentSetting("prune"); - const util::SettingsValue new_value = PruneSetting(prune_target_gb > 0, prune_target_gb); + const common::SettingsValue cur_value = node().getPersistentSetting("prune"); + const common::SettingsValue new_value = PruneSetting(prune_target_gb > 0, prune_target_gb); // Force setting to take effect. It is still safe to change the value at // this point because this function is only called after the intro screen is @@ -331,7 +331,7 @@ void OptionsModel::SetPruneTargetGB(int prune_target_gb) // Keep previous pruning size, if pruning was disabled. if (PruneEnabled(cur_value)) { - UpdateRwSetting(node(), Prune, "-prev", PruneEnabled(new_value) ? util::SettingsValue{} : cur_value); + UpdateRwSetting(node(), Prune, "-prev", PruneEnabled(new_value) ? common::SettingsValue{} : cur_value); } } @@ -457,7 +457,7 @@ QVariant OptionsModel::getOption(OptionID option, const std::string& suffix) con bool OptionsModel::setOption(OptionID option, const QVariant& value, const std::string& suffix) { auto changed = [&] { return value.isValid() && value != getOption(option, suffix); }; - auto update = [&](const util::SettingsValue& value) { return UpdateRwSetting(node(), option, suffix, value); }; + auto update = [&](const common::SettingsValue& value) { return UpdateRwSetting(node(), option, suffix, value); }; bool successful = true; /* set to false on parse error */ QSettings settings; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 76500a4a36..90aae0219e 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -10,6 +10,7 @@ #include <qt/forms/ui_debugwindow.h> #include <chainparams.h> +#include <common/system.h> #include <interfaces/node.h> #include <qt/bantablemodel.h> #include <qt/clientmodel.h> @@ -21,7 +22,6 @@ #include <rpc/server.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/threadnames.h> #include <univalue.h> @@ -249,7 +249,7 @@ bool RPCConsole::RPCParseCommandLine(interfaces::Node* node, std::string &strRes subelement = lastResult[parsed.value()]; } else if (lastResult.isObject()) - subelement = find_value(lastResult, curarg); + subelement = lastResult.find_value(curarg); else throw std::runtime_error("Invalid result query"); //no array or object: abort lastResult = subelement; @@ -448,8 +448,8 @@ void RPCExecutor::request(const QString &command, const WalletModel* wallet_mode { try // Nice formatting for standard-format error { - int code = find_value(objError, "code").getInt<int>(); - std::string message = find_value(objError, "message").get_str(); + int code = objError.find_value("code").getInt<int>(); + std::string message = objError.find_value("message").get_str(); Q_EMIT reply(RPCConsole::CMD_ERROR, QString::fromStdString(message) + " (code " + QString::number(code) + ")"); } catch (const std::runtime_error&) // raised when converting to invalid type, i.e. missing code or message @@ -744,7 +744,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ ui->dataDir->setText(model->dataDir()); ui->blocksDir->setText(model->blocksDir()); ui->startupTime->setText(model->formatClientStartupTime()); - ui->networkName->setText(QString::fromStdString(Params().NetworkIDString())); + ui->networkName->setText(QString::fromStdString(Params().GetChainTypeString())); //Setup autocomplete and attach it QStringList wordList; diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 096f8a0ded..8872f8be32 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -9,13 +9,13 @@ #include <qt/splashscreen.h> #include <clientversion.h> +#include <common/system.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <interfaces/wallet.h> #include <qt/guiutil.h> #include <qt/networkstyle.h> #include <qt/walletmodel.h> -#include <util/system.h> #include <util/translation.h> #include <functional> diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 5706964cc9..f17a6b7f74 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -19,6 +19,7 @@ #include <key.h> #include <key_io.h> #include <wallet/wallet.h> +#include <wallet/test/util.h> #include <walletinitinterface.h> #include <chrono> @@ -31,7 +32,7 @@ using wallet::AddWallet; using wallet::CWallet; -using wallet::CreateMockWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::RemoveWallet; using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WalletContext; @@ -75,7 +76,7 @@ void TestAddAddressesToSendBook(interfaces::Node& node) auto wallet_loader = interfaces::MakeWalletLoader(*test.m_node.chain, *Assert(test.m_node.args)); test.m_node.wallet_loader = wallet_loader.get(); node.setContext(&test.m_node); - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockableWalletDatabase()); wallet->LoadWallet(); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); { diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 000bbe65be..e918e84184 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -73,7 +73,7 @@ void AppTests::appTests() qRegisterMetaType<interfaces::BlockAndHeaderTipInfo>("interfaces::BlockAndHeaderTipInfo"); m_app.parameterSetup(); QVERIFY(m_app.createOptionsModel(/*resetSettings=*/true)); - QScopedPointer<const NetworkStyle> style(NetworkStyle::instantiate(Params().NetworkIDString())); + QScopedPointer<const NetworkStyle> style(NetworkStyle::instantiate(Params().GetChainType())); m_app.setupPlatformStyle(); m_app.createWindow(style.data()); connect(&m_app, &BitcoinApplication::windowShown, this, &AppTests::guiTests); diff --git a/src/qt/test/optiontests.cpp b/src/qt/test/optiontests.cpp index 0838e21678..e5a5179d87 100644 --- a/src/qt/test/optiontests.cpp +++ b/src/qt/test/optiontests.cpp @@ -18,13 +18,13 @@ OptionTests::OptionTests(interfaces::Node& node) : m_node(node) { - gArgs.LockSettings([&](util::Settings& s) { m_previous_settings = s; }); + gArgs.LockSettings([&](common::Settings& s) { m_previous_settings = s; }); } void OptionTests::init() { // reset args - gArgs.LockSettings([&](util::Settings& s) { s = m_previous_settings; }); + gArgs.LockSettings([&](common::Settings& s) { s = m_previous_settings; }); gArgs.ClearPathCache(); } @@ -76,14 +76,14 @@ void OptionTests::integerGetArgBug() // Test regression https://github.com/bitcoin/bitcoin/issues/24457. Ensure // that setting integer prune value doesn't cause an exception to be thrown // in the OptionsModel constructor - gArgs.LockSettings([&](util::Settings& settings) { + gArgs.LockSettings([&](common::Settings& settings) { settings.forced_settings.erase("prune"); settings.rw_settings["prune"] = 3814; }); gArgs.WriteSettingsFile(); bilingual_str error; QVERIFY(OptionsModel{m_node}.Init(error)); - gArgs.LockSettings([&](util::Settings& settings) { + gArgs.LockSettings([&](common::Settings& settings) { settings.rw_settings.erase("prune"); }); gArgs.WriteSettingsFile(); @@ -95,7 +95,7 @@ void OptionTests::parametersInteraction() // It was fixed via https://github.com/bitcoin-core/gui/pull/568. // With fListen=false in ~/.config/Bitcoin/Bitcoin-Qt.conf and all else left as default, // bitcoin-qt should set both -listen and -listenonion to false and start successfully. - gArgs.LockSettings([&](util::Settings& s) { + gArgs.LockSettings([&](common::Settings& s) { s.forced_settings.erase("listen"); s.forced_settings.erase("listenonion"); }); diff --git a/src/qt/test/optiontests.h b/src/qt/test/optiontests.h index 0c458c97a6..2d7b94933f 100644 --- a/src/qt/test/optiontests.h +++ b/src/qt/test/optiontests.h @@ -5,9 +5,9 @@ #ifndef BITCOIN_QT_TEST_OPTIONTESTS_H #define BITCOIN_QT_TEST_OPTIONTESTS_H +#include <common/settings.h> #include <qt/optionsmodel.h> #include <univalue.h> -#include <util/settings.h> #include <QObject> @@ -26,7 +26,7 @@ private Q_SLOTS: private: interfaces::Node& m_node; - util::Settings m_previous_settings; + common::Settings m_previous_settings; }; #endif // BITCOIN_QT_TEST_OPTIONTESTS_H diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 669a05fe0f..72e8055425 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -4,12 +4,12 @@ #include <qt/test/rpcnestedtests.h> +#include <common/system.h> #include <interfaces/node.h> -#include <rpc/server.h> #include <qt/rpcconsole.h> +#include <rpc/server.h> #include <test/util/setup_common.h> #include <univalue.h> -#include <util/system.h> #include <QTest> diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 2d069f76a0..e45fc1ced8 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -14,6 +14,7 @@ #include <qt/test/rpcnestedtests.h> #include <qt/test/uritests.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #ifdef ENABLE_WALLET #include <qt/test/addressbooktests.h> @@ -57,7 +58,7 @@ int main(int argc, char* argv[]) // // All tests must use their own testing setup (if needed). fs::create_directories([] { - BasicTestingSetup dummy{CBaseChainParams::REGTEST}; + BasicTestingSetup dummy{ChainType::REGTEST}; return gArgs.GetDataDirNet() / "blocks"; }()); @@ -70,6 +71,9 @@ int main(int argc, char* argv[]) gArgs.ForceSetArg("-upnp", "0"); gArgs.ForceSetArg("-natpmp", "0"); + std::string error; + if (!gArgs.ReadConfigFiles(error, true)) QWARN(error.c_str()); + // Prefer the "minimal" platform for the test instead of the normal default // platform ("xcb", "windows", or "cocoa") so tests can't unintentionally // interfere with any background GUIs and don't require extra resources. diff --git a/src/qt/test/util.cpp b/src/qt/test/util.cpp index c5ed20d967..d91b90fbba 100644 --- a/src/qt/test/util.cpp +++ b/src/qt/test/util.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <qt/test/util.h> + #include <chrono> #include <QApplication> diff --git a/src/qt/test/util.h b/src/qt/test/util.h index 13170c89ea..a1788ca74e 100644 --- a/src/qt/test/util.h +++ b/src/qt/test/util.h @@ -7,6 +7,8 @@ #include <chrono> +#include <qglobal.h> + QT_BEGIN_NAMESPACE class QString; QT_END_NAMESPACE diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 072718fe15..5f789e400e 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -26,6 +26,7 @@ #include <qt/walletmodel.h> #include <test/util/setup_common.h> #include <validation.h> +#include <wallet/test/util.h> #include <wallet/wallet.h> #include <chrono> @@ -46,7 +47,7 @@ using wallet::AddWallet; using wallet::CWallet; -using wallet::CreateMockWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::RemoveWallet; using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WALLET_FLAG_DISABLE_PRIVATE_KEYS; @@ -189,7 +190,7 @@ void SyncUpWallet(const std::shared_ptr<CWallet>& wallet, interfaces::Node& node std::shared_ptr<CWallet> SetupLegacyWatchOnlyWallet(interfaces::Node& node, TestChain100Setup& test) { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockableWalletDatabase()); wallet->LoadWallet(); { LOCK(wallet->cs_wallet); @@ -207,7 +208,7 @@ std::shared_ptr<CWallet> SetupLegacyWatchOnlyWallet(interfaces::Node& node, Test std::shared_ptr<CWallet> SetupDescriptorsWallet(interfaces::Node& node, TestChain100Setup& test) { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockableWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 2461a06930..fa110cfbc9 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -13,12 +13,12 @@ #include <qt/paymentserver.h> #include <qt/transactionrecord.h> +#include <common/system.h> #include <consensus/consensus.h> #include <interfaces/node.h> #include <interfaces/wallet.h> #include <key_io.h> #include <policy/policy.h> -#include <util/system.h> #include <validation.h> #include <wallet/types.h> diff --git a/src/random.cpp b/src/random.cpp index f4c51574cc..54500e6cc6 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -28,14 +28,10 @@ #include <sys/time.h> #endif -#ifdef HAVE_SYS_GETRANDOM -#include <sys/syscall.h> -#include <linux/random.h> -#endif -#if defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) -#include <unistd.h> +#if defined(HAVE_GETRANDOM) || (defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX)) #include <sys/random.h> #endif + #ifdef HAVE_SYSCTL_ARND #include <sys/sysctl.h> #endif @@ -252,7 +248,7 @@ static void Strengthen(const unsigned char (&seed)[32], SteadyClock::duration du /** Fallback: get 32 bytes of system entropy from /dev/urandom. The most * compatible way to get cryptographic randomness on UNIX-ish platforms. */ -static void GetDevURandom(unsigned char *ent32) +[[maybe_unused]] static void GetDevURandom(unsigned char *ent32) { int f = open("/dev/urandom", O_RDONLY); if (f == -1) { @@ -285,23 +281,14 @@ void GetOSRand(unsigned char *ent32) RandFailure(); } CryptReleaseContext(hProvider, 0); -#elif defined(HAVE_SYS_GETRANDOM) +#elif defined(HAVE_GETRANDOM) /* Linux. From the getrandom(2) man page: * "If the urandom source has been initialized, reads of up to 256 bytes * will always return as many bytes as requested and will not be * interrupted by signals." */ - int rv = syscall(SYS_getrandom, ent32, NUM_OS_RANDOM_BYTES, 0); - if (rv != NUM_OS_RANDOM_BYTES) { - if (rv < 0 && errno == ENOSYS) { - /* Fallback for kernel <3.17: the return value will be -1 and errno - * ENOSYS if the syscall is not available, in that case fall back - * to /dev/urandom. - */ - GetDevURandom(ent32); - } else { - RandFailure(); - } + if (getrandom(ent32, NUM_OS_RANDOM_BYTES, 0) != NUM_OS_RANDOM_BYTES) { + RandFailure(); } #elif defined(__OpenBSD__) /* OpenBSD. From the arc4random(3) man page: @@ -311,16 +298,10 @@ void GetOSRand(unsigned char *ent32) The function call is always successful. */ arc4random_buf(ent32, NUM_OS_RANDOM_BYTES); - // Silence a compiler warning about unused function. - (void)GetDevURandom; #elif defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) - /* getentropy() is available on macOS 10.12 and later. - */ if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { RandFailure(); } - // Silence a compiler warning about unused function. - (void)GetDevURandom; #elif defined(HAVE_SYSCTL_ARND) /* FreeBSD, NetBSD and similar. It is possible for the call to return less * bytes than requested, so need to read in a loop. @@ -334,8 +315,6 @@ void GetOSRand(unsigned char *ent32) } have += len; } while (have < NUM_OS_RANDOM_BYTES); - // Silence a compiler warning about unused function. - (void)GetDevURandom; #else /* Fall back to /dev/urandom if there is no specific method implemented to * get system entropy for this OS. diff --git a/src/rest.cpp b/src/rest.cpp index e46406f1ad..ba149c1a9e 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -24,8 +24,8 @@ #include <streams.h> #include <sync.h> #include <txmempool.h> +#include <util/any.h> #include <util/check.h> -#include <util/system.h> #include <validation.h> #include <version.h> @@ -36,7 +36,6 @@ using node::GetTransaction; using node::NodeContext; -using node::ReadBlockFromDisk; static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000; @@ -311,7 +310,7 @@ static bool rest_block(const std::any& context, } - if (!ReadBlockFromDisk(block, pblockindex, chainman.GetParams().GetConsensus())) { + if (!chainman.m_blockman.ReadBlockFromDisk(block, *pblockindex)) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } @@ -628,7 +627,7 @@ static bool rest_deploymentinfo(const std::any& context, HTTPRequest* req, const return RESTERR(req, HTTP_BAD_REQUEST, "Block not found"); } - jsonRequest.params.pushKV("blockhash", hash_str); + jsonRequest.params.push_back(hash_str); } req->WriteHeader("Content-Type", "application/json"); @@ -716,7 +715,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string const NodeContext* const node = GetNodeContext(context, req); if (!node) return false; uint256 hashBlock = uint256(); - const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, node->mempool.get(), hash, Params().GetConsensus(), hashBlock); + const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, node->mempool.get(), hash, hashBlock, node->chainman->m_blockman); if (!tx) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 11484a9d8d..ee3237638e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -58,9 +58,7 @@ using kernel::CoinStatsHashType; using node::BlockManager; using node::NodeContext; -using node::ReadBlockFromDisk; using node::SnapshotMetadata; -using node::UndoReadFromDisk; struct CUpdatedBlock { @@ -183,7 +181,7 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: CBlockUndo blockUndo; const bool is_not_pruned{WITH_LOCK(::cs_main, return !blockman.IsBlockPruned(blockindex))}; - const bool have_undo{is_not_pruned && UndoReadFromDisk(blockUndo, blockindex)}; + const bool have_undo{is_not_pruned && blockman.UndoReadFromDisk(blockUndo, *blockindex)}; for (size_t i = 0; i < block.vtx.size(); ++i) { const CTransactionRef& tx = block.vtx.at(i); @@ -430,7 +428,7 @@ static RPCHelpMan getblockfrompeer() "getblockfrompeer", "Attempt to fetch block from a given peer.\n\n" "We must have the header for this block, e.g. using submitheader.\n" - "Subsequent calls for the same block and a new peer will cause the response from the previous peer to be ignored.\n" + "Subsequent calls for the same block may cause the response from the previous peer to be ignored.\n" "Peers generally ignore requests for a stale block that they never fully verified, or one that is more than a month old.\n" "When a peer does not respond with a block, we will disconnect.\n" "Note: The block could be re-pruned as soon as it is received.\n\n" @@ -587,7 +585,7 @@ static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex* pblocki } } - if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { + if (!blockman.ReadBlockFromDisk(block, *pblockindex)) { // Block not found on disk. This could be because we have the block // header in our index but not yet have the block or did not accept the // block. Or if the block was pruned right after we released the lock above. @@ -611,7 +609,7 @@ static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex* pblo } } - if (!UndoReadFromDisk(blockUndo, pblockindex)) { + if (!blockman.UndoReadFromDisk(blockUndo, *pblockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Can't read undo data from disk"); } @@ -1125,7 +1123,7 @@ static RPCHelpMan verifychain() LOCK(cs_main); Chainstate& active_chainstate = chainman.ActiveChainstate(); - return CVerifyDB().VerifyDB( + return CVerifyDB(chainman.GetNotifications()).VerifyDB( active_chainstate, chainman.GetParams().GetConsensus(), active_chainstate.CoinsTip(), check_level, check_depth) == VerifyDBResult::SUCCESS; }, }; @@ -1256,7 +1254,7 @@ RPCHelpMan getblockchaininfo() const CBlockIndex& tip{*CHECK_NONFATAL(active_chainstate.m_chain.Tip())}; const int height{tip.nHeight}; UniValue obj(UniValue::VOBJ); - obj.pushKV("chain", chainman.GetParams().NetworkIDString()); + obj.pushKV("chain", chainman.GetParams().GetChainTypeString()); obj.pushKV("blocks", height); obj.pushKV("headers", chainman.m_best_header ? chainman.m_best_header->nHeight : -1); obj.pushKV("bestblockhash", tip.GetBlockHash().GetHex()); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index f3c19003ff..edc0fb05d7 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -101,6 +101,11 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listunspent", 2, "addresses" }, { "listunspent", 3, "include_unsafe" }, { "listunspent", 4, "query_options" }, + { "listunspent", 4, "minimumAmount" }, + { "listunspent", 4, "maximumAmount" }, + { "listunspent", 4, "maximumCount" }, + { "listunspent", 4, "minimumSumAmount" }, + { "listunspent", 4, "include_immature_coinbase" }, { "getblock", 1, "verbosity" }, { "getblock", 1, "verbose" }, { "getblockheader", 1, "verbose" }, @@ -124,15 +129,45 @@ static const CRPCConvertParam vRPCConvertParams[] = { "submitpackage", 0, "package" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, + { "fundrawtransaction", 1, "add_inputs"}, + { "fundrawtransaction", 1, "include_unsafe"}, + { "fundrawtransaction", 1, "minconf"}, + { "fundrawtransaction", 1, "maxconf"}, + { "fundrawtransaction", 1, "changePosition"}, + { "fundrawtransaction", 1, "includeWatching"}, + { "fundrawtransaction", 1, "lockUnspents"}, + { "fundrawtransaction", 1, "fee_rate"}, + { "fundrawtransaction", 1, "feeRate"}, + { "fundrawtransaction", 1, "subtractFeeFromOutputs"}, + { "fundrawtransaction", 1, "input_weights"}, + { "fundrawtransaction", 1, "conf_target"}, + { "fundrawtransaction", 1, "replaceable"}, + { "fundrawtransaction", 1, "solving_data"}, { "fundrawtransaction", 2, "iswitness" }, { "walletcreatefundedpsbt", 0, "inputs" }, { "walletcreatefundedpsbt", 1, "outputs" }, { "walletcreatefundedpsbt", 2, "locktime" }, { "walletcreatefundedpsbt", 3, "options" }, + { "walletcreatefundedpsbt", 3, "add_inputs"}, + { "walletcreatefundedpsbt", 3, "include_unsafe"}, + { "walletcreatefundedpsbt", 3, "minconf"}, + { "walletcreatefundedpsbt", 3, "maxconf"}, + { "walletcreatefundedpsbt", 3, "changePosition"}, + { "walletcreatefundedpsbt", 3, "includeWatching"}, + { "walletcreatefundedpsbt", 3, "lockUnspents"}, + { "walletcreatefundedpsbt", 3, "fee_rate"}, + { "walletcreatefundedpsbt", 3, "feeRate"}, + { "walletcreatefundedpsbt", 3, "subtractFeeFromOutputs"}, + { "walletcreatefundedpsbt", 3, "conf_target"}, + { "walletcreatefundedpsbt", 3, "replaceable"}, + { "walletcreatefundedpsbt", 3, "solving_data"}, { "walletcreatefundedpsbt", 4, "bip32derivs" }, { "walletprocesspsbt", 1, "sign" }, { "walletprocesspsbt", 3, "bip32derivs" }, { "walletprocesspsbt", 4, "finalize" }, + { "descriptorprocesspsbt", 1, "descriptors"}, + { "descriptorprocesspsbt", 3, "bip32derivs" }, + { "descriptorprocesspsbt", 4, "finalize" }, { "createpsbt", 0, "inputs" }, { "createpsbt", 1, "outputs" }, { "createpsbt", 2, "locktime" }, @@ -154,18 +189,49 @@ static const CRPCConvertParam vRPCConvertParams[] = { "send", 1, "conf_target" }, { "send", 3, "fee_rate"}, { "send", 4, "options" }, + { "send", 4, "add_inputs"}, + { "send", 4, "include_unsafe"}, + { "send", 4, "minconf"}, + { "send", 4, "maxconf"}, + { "send", 4, "add_to_wallet"}, + { "send", 4, "change_position"}, + { "send", 4, "fee_rate"}, + { "send", 4, "include_watching"}, + { "send", 4, "inputs"}, + { "send", 4, "locktime"}, + { "send", 4, "lock_unspents"}, + { "send", 4, "psbt"}, + { "send", 4, "subtract_fee_from_outputs"}, + { "send", 4, "conf_target"}, + { "send", 4, "replaceable"}, + { "send", 4, "solving_data"}, { "sendall", 0, "recipients" }, { "sendall", 1, "conf_target" }, { "sendall", 3, "fee_rate"}, { "sendall", 4, "options" }, + { "sendall", 4, "add_to_wallet"}, + { "sendall", 4, "fee_rate"}, + { "sendall", 4, "include_watching"}, + { "sendall", 4, "inputs"}, + { "sendall", 4, "locktime"}, + { "sendall", 4, "lock_unspents"}, + { "sendall", 4, "psbt"}, + { "sendall", 4, "send_max"}, + { "sendall", 4, "minconf"}, + { "sendall", 4, "maxconf"}, + { "sendall", 4, "conf_target"}, + { "sendall", 4, "replaceable"}, + { "sendall", 4, "solving_data"}, { "simulaterawtransaction", 0, "rawtxs" }, { "simulaterawtransaction", 1, "options" }, + { "simulaterawtransaction", 1, "include_watchonly"}, { "importprivkey", 2, "rescan" }, { "importaddress", 2, "rescan" }, { "importaddress", 3, "p2sh" }, { "importpubkey", 2, "rescan" }, { "importmulti", 0, "requests" }, { "importmulti", 1, "options" }, + { "importmulti", 1, "rescan" }, { "importdescriptors", 0, "requests" }, { "listdescriptors", 0, "private" }, { "verifychain", 0, "checklevel" }, @@ -189,7 +255,15 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getmempooldescendants", 1, "verbose" }, { "gettxspendingprevout", 0, "outputs" }, { "bumpfee", 1, "options" }, + { "bumpfee", 1, "conf_target"}, + { "bumpfee", 1, "fee_rate"}, + { "bumpfee", 1, "replaceable"}, + { "bumpfee", 1, "outputs"}, { "psbtbumpfee", 1, "options" }, + { "psbtbumpfee", 1, "conf_target"}, + { "psbtbumpfee", 1, "fee_rate"}, + { "psbtbumpfee", 1, "replaceable"}, + { "psbtbumpfee", 1, "outputs"}, { "logging", 0, "include" }, { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, @@ -223,6 +297,14 @@ static const CRPCConvertParam vRPCConvertParams[] = }; // clang-format on +/** Parse string to UniValue or throw runtime_error if string contains invalid JSON */ +static UniValue Parse(std::string_view raw) +{ + UniValue parsed; + if (!parsed.read(raw)) throw std::runtime_error(tfm::format("Error parsing JSON: %s", raw)); + return parsed; +} + class CRPCConvertTable { private: @@ -235,13 +317,13 @@ public: /** Return arg_value as UniValue, and first parse it if it is a non-string parameter */ UniValue ArgToUniValue(std::string_view arg_value, const std::string& method, int param_idx) { - return members.count({method, param_idx}) > 0 ? ParseNonRFCJSONValue(arg_value) : arg_value; + return members.count({method, param_idx}) > 0 ? Parse(arg_value) : arg_value; } /** Return arg_value as UniValue, and first parse it if it is a non-string parameter */ UniValue ArgToUniValue(std::string_view arg_value, const std::string& method, const std::string& param_name) { - return membersByName.count({method, param_name}) > 0 ? ParseNonRFCJSONValue(arg_value) : arg_value; + return membersByName.count({method, param_name}) > 0 ? Parse(arg_value) : arg_value; } }; @@ -255,16 +337,6 @@ CRPCConvertTable::CRPCConvertTable() static CRPCConvertTable rpcCvtTable; -/** Non-RFC4627 JSON parser, accepts internal values (such as numbers, true, false, null) - * as well as objects and arrays. - */ -UniValue ParseNonRFCJSONValue(std::string_view raw) -{ - UniValue parsed; - if (!parsed.read(raw)) throw std::runtime_error(tfm::format("Error parsing JSON: %s", raw)); - return parsed; -} - UniValue RPCConvertValues(const std::string &strMethod, const std::vector<std::string> &strParams) { UniValue params(UniValue::VARR); diff --git a/src/rpc/client.h b/src/rpc/client.h index 3c5c4fc4d6..b67cd27fdf 100644 --- a/src/rpc/client.h +++ b/src/rpc/client.h @@ -17,9 +17,4 @@ UniValue RPCConvertValues(const std::string& strMethod, const std::vector<std::s /** Convert named arguments to command-specific RPC representation */ UniValue RPCConvertNamedValues(const std::string& strMethod, const std::vector<std::string>& strParams); -/** Non-RFC4627 JSON parser, accepts internal values (such as numbers, true, false, null) - * as well as objects and arrays. - */ -UniValue ParseNonRFCJSONValue(std::string_view raw); - #endif // BITCOIN_RPC_CLIENT_H diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index 1e139c9990..310eec5f15 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -2,14 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <chainparamsbase.h> #include <common/args.h> +#include <common/system.h> #include <external_signer.h> #include <rpc/protocol.h> #include <rpc/server.h> #include <rpc/util.h> #include <util/strencodings.h> -#include <util/system.h> #include <string> #include <vector> @@ -43,7 +42,7 @@ static RPCHelpMan enumeratesigners() { const std::string command = gArgs.GetArg("-signer", ""); if (command == "") throw JSONRPCError(RPC_MISC_ERROR, "Error: restart bitcoind with -signer=<cmd>"); - const std::string chain = gArgs.GetChainName(); + const std::string chain = gArgs.GetChainTypeString(); UniValue signers_res = UniValue::VARR; try { std::vector<ExternalSigner> signers; diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 927b4ce1fc..89c403b6f5 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -638,7 +638,7 @@ static RPCHelpMan gettxspendingprevout() }, /*fAllowNull=*/false, /*fStrict=*/true); const uint256 txid(ParseHashO(o, "txid")); - const int nOutput{find_value(o, "vout").getInt<int>()}; + const int nOutput{o.find_value("vout").getInt<int>()}; if (nOutput < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout cannot be negative"); } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index d55e20ba5e..074cecadd2 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -5,6 +5,7 @@ #include <chain.h> #include <chainparams.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> @@ -32,7 +33,6 @@ #include <univalue.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -435,7 +435,7 @@ static RPCHelpMan getmininginfo() obj.pushKV("difficulty", (double)GetDifficulty(active_chain.Tip())); obj.pushKV("networkhashps", getnetworkhashps().HandleRequest(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); - obj.pushKV("chain", chainman.GetParams().NetworkIDString()); + obj.pushKV("chain", chainman.GetParams().GetChainTypeString()); obj.pushKV("warnings", GetWarnings(false).original); return obj; }, @@ -480,6 +480,40 @@ static RPCHelpMan prioritisetransaction() }; } +static RPCHelpMan getprioritisedtransactions() +{ + return RPCHelpMan{"getprioritisedtransactions", + "Returns a map of all user-created (see prioritisetransaction) fee deltas by txid, and whether the tx is present in mempool.", + {}, + RPCResult{ + RPCResult::Type::OBJ_DYN, "prioritisation-map", "prioritisation keyed by txid", + { + {RPCResult::Type::OBJ, "txid", "", { + {RPCResult::Type::NUM, "fee_delta", "transaction fee delta in satoshis"}, + {RPCResult::Type::BOOL, "in_mempool", "whether this transaction is currently in mempool"}, + }} + }, + }, + RPCExamples{ + HelpExampleCli("getprioritisedtransactions", "") + + HelpExampleRpc("getprioritisedtransactions", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + NodeContext& node = EnsureAnyNodeContext(request.context); + CTxMemPool& mempool = EnsureMemPool(node); + UniValue rpc_result{UniValue::VOBJ}; + for (const auto& delta_info : mempool.GetPrioritisedTransactions()) { + UniValue result_inner{UniValue::VOBJ}; + result_inner.pushKV("fee_delta", delta_info.delta); + result_inner.pushKV("in_mempool", delta_info.in_mempool); + rpc_result.pushKV(delta_info.txid.GetHex(), result_inner); + } + return rpc_result; + }, + }; +} + // NOTE: Assumes a conclusive result; if result is inconclusive, it must be handled by caller static UniValue BIP22ValidationResult(const BlockValidationState& state) @@ -612,7 +646,7 @@ static RPCHelpMan getblocktemplate() if (!request.params[0].isNull()) { const UniValue& oparam = request.params[0].get_obj(); - const UniValue& modeval = find_value(oparam, "mode"); + const UniValue& modeval = oparam.find_value("mode"); if (modeval.isStr()) strMode = modeval.get_str(); else if (modeval.isNull()) @@ -621,11 +655,11 @@ static RPCHelpMan getblocktemplate() } else throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode"); - lpval = find_value(oparam, "longpollid"); + lpval = oparam.find_value("longpollid"); if (strMode == "proposal") { - const UniValue& dataval = find_value(oparam, "data"); + const UniValue& dataval = oparam.find_value("data"); if (!dataval.isStr()) throw JSONRPCError(RPC_TYPE_ERROR, "Missing data String key for proposal"); @@ -652,7 +686,7 @@ static RPCHelpMan getblocktemplate() return BIP22ValidationResult(state); } - const UniValue& aClientRules = find_value(oparam, "rules"); + const UniValue& aClientRules = oparam.find_value("rules"); if (aClientRules.isArray()) { for (unsigned int i = 0; i < aClientRules.size(); ++i) { const UniValue& v = aClientRules[i]; @@ -1048,6 +1082,7 @@ void RegisterMiningRPCCommands(CRPCTable& t) {"mining", &getnetworkhashps}, {"mining", &getmininginfo}, {"mining", &prioritisetransaction}, + {"mining", &getprioritisedtransactions}, {"mining", &getblocktemplate}, {"mining", &submitblock}, {"mining", &submitheader}, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 7ffa777ef4..a2a46ef32f 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -21,6 +21,7 @@ #include <rpc/util.h> #include <sync.h> #include <timedata.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> @@ -354,7 +355,7 @@ static RPCHelpMan addconnection() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - if (Params().NetworkIDString() != CBaseChainParams::REGTEST) { + if (Params().GetChainType() != ChainType::REGTEST) { throw std::runtime_error("addconnection is for regression testing (-regtest mode) only."); } @@ -512,15 +513,15 @@ static RPCHelpMan getaddednodeinfo() static RPCHelpMan getnettotals() { return RPCHelpMan{"getnettotals", - "\nReturns information about network traffic, including bytes in, bytes out,\n" - "and current time.\n", - {}, + "Returns information about network traffic, including bytes in, bytes out,\n" + "and current system time.", + {}, RPCResult{ RPCResult::Type::OBJ, "", "", { {RPCResult::Type::NUM, "totalbytesrecv", "Total bytes received"}, {RPCResult::Type::NUM, "totalbytessent", "Total bytes sent"}, - {RPCResult::Type::NUM_TIME, "timemillis", "Current " + UNIX_EPOCH_TIME + " in milliseconds"}, + {RPCResult::Type::NUM_TIME, "timemillis", "Current system " + UNIX_EPOCH_TIME + " in milliseconds"}, {RPCResult::Type::OBJ, "uploadtarget", "", { {RPCResult::Type::NUM, "timeframe", "Length of the measuring timeframe in seconds"}, @@ -544,7 +545,7 @@ static RPCHelpMan getnettotals() UniValue obj(UniValue::VOBJ); obj.pushKV("totalbytesrecv", connman.GetTotalBytesRecv()); obj.pushKV("totalbytessent", connman.GetTotalBytesSent()); - obj.pushKV("timemillis", GetTimeMillis()); + obj.pushKV("timemillis", TicksSinceEpoch<std::chrono::milliseconds>(SystemClock::now())); UniValue outboundLimit(UniValue::VOBJ); outboundLimit.pushKV("timeframe", count_seconds(connman.GetMaxOutboundTimeframe())); @@ -712,9 +713,10 @@ static RPCHelpMan setban() isSubnet = true; if (!isSubnet) { - CNetAddr resolved; - LookupHost(request.params[0].get_str(), resolved, false); - netAddr = resolved; + const std::optional<CNetAddr> addr{LookupHost(request.params[0].get_str(), false)}; + if (addr.has_value()) { + netAddr = addr.value(); + } } else LookupSubNet(request.params[0].get_str(), subNet); @@ -942,11 +944,11 @@ static RPCHelpMan addpeeraddress() const bool tried{request.params[2].isNull() ? false : request.params[2].get_bool()}; UniValue obj(UniValue::VOBJ); - CNetAddr net_addr; + std::optional<CNetAddr> net_addr{LookupHost(addr_string, false)}; bool success{false}; - if (LookupHost(addr_string, net_addr, false)) { - CService service{net_addr, port}; + if (net_addr.has_value()) { + CService service{net_addr.value(), port}; CAddress address{MaybeFlipIPv6toCJDNS(service), ServiceFlags{NODE_NETWORK | NODE_WITNESS}}; address.nTime = Now<NodeSeconds>(); // The source address is set equal to the address. This is equivalent to the peer diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 5918bc6e38..45d46d223b 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -19,9 +19,9 @@ #include <rpc/util.h> #include <scheduler.h> #include <univalue.h> +#include <util/any.h> #include <util/check.h> #include <util/syscall_sandbox.h> -#include <util/system.h> #include <stdint.h> #ifdef HAVE_MALLOC_INFO @@ -163,7 +163,7 @@ static RPCHelpMan getmemoryinfo() { {"mode", RPCArg::Type::STR, RPCArg::Default{"stats"}, "determines what kind of information is returned.\n" " - \"stats\" returns general statistics about memory usage in the daemon.\n" - " - \"mallocinfo\" returns an XML string describing low-level heap state (only available if compiled with glibc 2.10+)."}, + " - \"mallocinfo\" returns an XML string describing low-level heap state (only available if compiled with glibc)."}, }, { RPCResult{"mode \"stats\"", diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 4a918cbd42..eb0200ccf5 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -51,8 +51,6 @@ using node::FindCoins; using node::GetTransaction; using node::NodeContext; using node::PSBTAnalysis; -using node::ReadBlockFromDisk; -using node::UndoReadFromDisk; static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, Chainstate& active_chainstate, const CTxUndo* txundo = nullptr, @@ -172,8 +170,9 @@ static std::vector<RPCArg> CreateTxDoc() }; } -// Update PSBT with information from the mempool, the UTXO set, the txindex, and the provided descriptors -PartiallySignedTransaction ProcessPSBT(const std::string& psbt_string, const std::any& context, const HidingSigningProvider& provider) +// Update PSBT with information from the mempool, the UTXO set, the txindex, and the provided descriptors. +// Optionally, sign the inputs that we can using information from the descriptors. +PartiallySignedTransaction ProcessPSBT(const std::string& psbt_string, const std::any& context, const HidingSigningProvider& provider, int sighash_type, bool finalize) { // Unserialize the transactions PartiallySignedTransaction psbtx; @@ -242,9 +241,10 @@ PartiallySignedTransaction ProcessPSBT(const std::string& psbt_string, const std } // Update script/keypath information using descriptor data. - // Note that SignPSBTInput does a lot more than just constructing ECDSA signatures - // we don't actually care about those here, in fact. - SignPSBTInput(provider, psbtx, /*index=*/i, &txdata, /*sighash=*/1); + // Note that SignPSBTInput does a lot more than just constructing ECDSA signatures. + // We only actually care about those if our signing provider doesn't hide private + // information, as is the case with `descriptorprocesspsbt` + SignPSBTInput(provider, psbtx, /*index=*/i, &txdata, sighash_type, /*out_sigdata=*/nullptr, finalize); } // Update script/keypath information using descriptor data. @@ -362,7 +362,7 @@ static RPCHelpMan getrawtransaction() } uint256 hash_block; - const CTransactionRef tx = GetTransaction(blockindex, node.mempool.get(), hash, chainman.GetConsensus(), hash_block); + const CTransactionRef tx = GetTransaction(blockindex, node.mempool.get(), hash, hash_block, chainman.m_blockman); if (!tx) { std::string errmsg; if (blockindex) { @@ -406,7 +406,7 @@ static RPCHelpMan getrawtransaction() if (tx->IsCoinBase() || !blockindex || is_block_pruned || - !(UndoReadFromDisk(blockUndo, blockindex) && ReadBlockFromDisk(block, blockindex, Params().GetConsensus()))) { + !(chainman.m_blockman.UndoReadFromDisk(blockUndo, *blockindex) && chainman.m_blockman.ReadBlockFromDisk(block, *blockindex))) { TxToJSON(*tx, hash_block, result, chainman.ActiveChainstate()); return result; } @@ -1697,7 +1697,9 @@ static RPCHelpMan utxoupdatepsbt() const PartiallySignedTransaction& psbtx = ProcessPSBT( request.params[0].get_str(), request.context, - HidingSigningProvider(&provider, /*hide_secret=*/true, /*hide_origin=*/false)); + HidingSigningProvider(&provider, /*hide_secret=*/true, /*hide_origin=*/false), + /*sighash_type=*/SIGHASH_ALL, + /*finalize=*/false); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; @@ -1916,6 +1918,82 @@ static RPCHelpMan analyzepsbt() }; } +RPCHelpMan descriptorprocesspsbt() +{ + return RPCHelpMan{"descriptorprocesspsbt", + "\nUpdate all segwit inputs in a PSBT with information from output descriptors, the UTXO set or the mempool. \n" + "Then, sign the inputs we are able to with information from the output descriptors. ", + { + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction base64 string"}, + {"descriptors", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of either strings or objects", { + {"", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with an output descriptor and extra information", { + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, + {"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "Up to what index HD chains should be explored (either end or [begin,end])"}, + }}, + }}, + {"sighashtype", RPCArg::Type::STR, RPCArg::Default{"DEFAULT for Taproot, ALL otherwise"}, "The signature hash type to sign with if not specified by the PSBT. Must be one of\n" + " \"DEFAULT\"\n" + " \"ALL\"\n" + " \"NONE\"\n" + " \"SINGLE\"\n" + " \"ALL|ANYONECANPAY\"\n" + " \"NONE|ANYONECANPAY\"\n" + " \"SINGLE|ANYONECANPAY\""}, + {"bip32derivs", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include BIP 32 derivation paths for public keys if we know them"}, + {"finalize", RPCArg::Type::BOOL, RPCArg::Default{true}, "Also finalize inputs if possible"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "psbt", "The base64-encoded partially signed transaction"}, + {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, + } + }, + RPCExamples{ + HelpExampleCli("descriptorprocesspsbt", "\"psbt\" \"[\\\"descriptor1\\\", \\\"descriptor2\\\"]\"") + + HelpExampleCli("descriptorprocesspsbt", "\"psbt\" \"[{\\\"desc\\\":\\\"mydescriptor\\\", \\\"range\\\":21}]\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + // Add descriptor information to a signing provider + FlatSigningProvider provider; + + auto descs = request.params[1].get_array(); + for (size_t i = 0; i < descs.size(); ++i) { + EvalDescriptorStringOrObject(descs[i], provider, /*expand_priv=*/true); + } + + int sighash_type = ParseSighashString(request.params[2]); + bool bip32derivs = request.params[3].isNull() ? true : request.params[3].get_bool(); + bool finalize = request.params[4].isNull() ? true : request.params[4].get_bool(); + + const PartiallySignedTransaction& psbtx = ProcessPSBT( + request.params[0].get_str(), + request.context, + HidingSigningProvider(&provider, /*hide_secret=*/false, !bip32derivs), + sighash_type, + finalize); + + // Check whether or not all of the inputs are now signed + bool complete = true; + for (const auto& input : psbtx.inputs) { + complete &= PSBTInputSigned(input); + } + + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + + UniValue result(UniValue::VOBJ); + + result.pushKV("psbt", EncodeBase64(ssTx)); + result.pushKV("complete", complete); + + return result; +}, + }; +} + void RegisterRawTransactionRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ @@ -1931,6 +2009,7 @@ void RegisterRawTransactionRPCCommands(CRPCTable& t) {"rawtransactions", &createpsbt}, {"rawtransactions", &converttopsbt}, {"rawtransactions", &utxoupdatepsbt}, + {"rawtransactions", &descriptorprocesspsbt}, {"rawtransactions", &joinpsbts}, {"rawtransactions", &analyzepsbt}, }; diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index 3ba930f84f..3a6fa39e4d 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -36,7 +36,7 @@ void AddInputs(CMutableTransaction& rawTx, const UniValue& inputs_in, std::optio uint256 txid = ParseHashO(o, "txid"); - const UniValue& vout_v = find_value(o, "vout"); + const UniValue& vout_v = o.find_value("vout"); if (!vout_v.isNum()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, missing vout key"); int nOutput = vout_v.getInt<int>(); @@ -54,7 +54,7 @@ void AddInputs(CMutableTransaction& rawTx, const UniValue& inputs_in, std::optio } // set the sequence number if passed in the parameters object - const UniValue& sequenceObj = find_value(o, "sequence"); + const UniValue& sequenceObj = o.find_value("sequence"); if (sequenceObj.isNum()) { int64_t seqNr64 = sequenceObj.getInt<int64_t>(); if (seqNr64 < 0 || seqNr64 > CTxIn::SEQUENCE_FINAL) { @@ -187,7 +187,7 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst uint256 txid = ParseHashO(prevOut, "txid"); - int nOut = find_value(prevOut, "vout").getInt<int>(); + int nOut = prevOut.find_value("vout").getInt<int>(); if (nOut < 0) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "vout cannot be negative"); } @@ -208,7 +208,7 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst newcoin.out.scriptPubKey = scriptPubKey; newcoin.out.nValue = MAX_MONEY; if (prevOut.exists("amount")) { - newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); + newcoin.out.nValue = AmountFromValue(prevOut.find_value("amount")); } newcoin.nHeight = 1; coins[out] = std::move(newcoin); @@ -223,8 +223,8 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst {"redeemScript", UniValueType(UniValue::VSTR)}, {"witnessScript", UniValueType(UniValue::VSTR)}, }, true); - UniValue rs = find_value(prevOut, "redeemScript"); - UniValue ws = find_value(prevOut, "witnessScript"); + const UniValue& rs{prevOut.find_value("redeemScript")}; + const UniValue& ws{prevOut.find_value("witnessScript")}; if (rs.isNull() && ws.isNull()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing redeemScript/witnessScript"); } diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index ad91ed0f23..4c67da8b70 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -88,7 +88,7 @@ bool GenerateAuthCookie(std::string *cookie_out) std::string cookie = COOKIEAUTH_USER + ":" + HexStr(rand_pwd); /** the umask determines what permissions are used to create this file - - * these are set to 0077 in util/system.cpp. + * these are set to 0077 in common/system.cpp. */ std::ofstream file; fs::path filepath_tmp = GetAuthCookieFile(true); @@ -165,10 +165,10 @@ void JSONRPCRequest::parse(const UniValue& valRequest) const UniValue& request = valRequest.get_obj(); // Parse id now so errors from here on will have the id - id = find_value(request, "id"); + id = request.find_value("id"); // Parse method - UniValue valMethod = find_value(request, "method"); + const UniValue& valMethod{request.find_value("method")}; if (valMethod.isNull()) throw JSONRPCError(RPC_INVALID_REQUEST, "Missing method"); if (!valMethod.isStr()) @@ -181,7 +181,7 @@ void JSONRPCRequest::parse(const UniValue& valRequest) LogPrint(BCLog::RPC, "ThreadRPCServer method=%s user=%s\n", SanitizeString(strMethod), this->authUser); // Parse params - UniValue valParams = find_value(request, "params"); + const UniValue& valParams{request.find_value("params")}; if (valParams.isArray() || valParams.isObject()) params = valParams; else if (valParams.isNull()) diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 354f949002..d39efcb245 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -6,13 +6,13 @@ #include <rpc/server.h> #include <common/args.h> +#include <common/system.h> #include <logging.h> #include <rpc/util.h> #include <shutdown.h> #include <sync.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/time.h> #include <boost/signals2/signal.hpp> @@ -392,7 +392,7 @@ std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq) * Process named arguments into a vector of positional arguments, based on the * passed-in specification for the RPC call's arguments. */ -static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, const std::vector<std::string>& argNames) +static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, const std::vector<std::pair<std::string, bool>>& argNames) { JSONRPCRequest out = in; out.params = UniValue(UniValue::VARR); @@ -417,7 +417,9 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c // "args" parameter, if present. int hole = 0; int initial_hole_size = 0; - for (const std::string &argNamePattern: argNames) { + const std::string* initial_param = nullptr; + UniValue options{UniValue::VOBJ}; + for (const auto& [argNamePattern, named_only]: argNames) { std::vector<std::string> vargNames = SplitString(argNamePattern, '|'); auto fr = argsIn.end(); for (const std::string & argName : vargNames) { @@ -426,7 +428,22 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c break; } } - if (fr != argsIn.end()) { + + // Handle named-only parameters by pushing them into a temporary options + // object, and then pushing the accumulated options as the next + // positional argument. + if (named_only) { + if (fr != argsIn.end()) { + if (options.exists(fr->first)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + fr->first + " specified multiple times"); + } + options.__pushKV(fr->first, *fr->second); + argsIn.erase(fr); + } + continue; + } + + if (!options.empty() || fr != argsIn.end()) { for (int i = 0; i < hole; ++i) { // Fill hole between specified parameters with JSON nulls, // but not at the end (for backwards compatibility with calls @@ -434,12 +451,26 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c out.params.push_back(UniValue()); } hole = 0; - out.params.push_back(*fr->second); - argsIn.erase(fr); + if (!initial_param) initial_param = &argNamePattern; } else { hole += 1; if (out.params.empty()) initial_hole_size = hole; } + + // If named input parameter "fr" is present, push it onto out.params. If + // options are present, push them onto out.params. If both are present, + // throw an error. + if (fr != argsIn.end()) { + if (!options.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + fr->first + " conflicts with parameter " + options.getKeys().front()); + } + out.params.push_back(*fr->second); + argsIn.erase(fr); + } + if (!options.empty()) { + out.params.push_back(std::move(options)); + options = UniValue{UniValue::VOBJ}; + } } // If leftover "args" param was found, use it as a source of positional // arguments and add named arguments after. This is a convenience for @@ -447,9 +478,8 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c // arguments as described in doc/JSON-RPC-interface.md#parameter-passing auto positional_args{argsIn.extract("args")}; if (positional_args && positional_args.mapped()->isArray()) { - const bool has_named_arguments{initial_hole_size < (int)argNames.size()}; - if (initial_hole_size < (int)positional_args.mapped()->size() && has_named_arguments) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + argNames[initial_hole_size] + " specified twice both as positional and named argument"); + if (initial_hole_size < (int)positional_args.mapped()->size() && initial_param) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + *initial_param + " specified twice both as positional and named argument"); } // Assign positional_args to out.params and append named_args after. UniValue named_args{std::move(out.params)}; diff --git a/src/rpc/server.h b/src/rpc/server.h index 01e8556050..24658ddb8b 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -95,7 +95,7 @@ public: using Actor = std::function<bool(const JSONRPCRequest& request, UniValue& result, bool last_handler)>; //! Constructor taking Actor callback supporting multiple handlers. - CRPCCommand(std::string category, std::string name, Actor actor, std::vector<std::string> args, intptr_t unique_id) + CRPCCommand(std::string category, std::string name, Actor actor, std::vector<std::pair<std::string, bool>> args, intptr_t unique_id) : category(std::move(category)), name(std::move(name)), actor(std::move(actor)), argNames(std::move(args)), unique_id(unique_id) { @@ -115,7 +115,16 @@ public: std::string category; std::string name; Actor actor; - std::vector<std::string> argNames; + //! List of method arguments and whether they are named-only. Incoming RPC + //! requests contain a "params" field that can either be an array containing + //! unnamed arguments or an object containing named arguments. The + //! "argNames" vector is used in the latter case to transform the params + //! object into an array. Each argument in "argNames" gets mapped to a + //! unique position in the array, based on the order it is listed, unless + //! the argument is a named-only argument with argNames[x].second set to + //! true. Named-only arguments are combined into a JSON object that is + //! appended after other arguments, see transformNamedArguments for details. + std::vector<std::pair<std::string, bool>> argNames; intptr_t unique_id; }; diff --git a/src/rpc/server_util.cpp b/src/rpc/server_util.cpp index 13d007b496..1d4afb3758 100644 --- a/src/rpc/server_util.cpp +++ b/src/rpc/server_util.cpp @@ -11,7 +11,7 @@ #include <rpc/protocol.h> #include <rpc/request.h> #include <txmempool.h> -#include <util/system.h> +#include <util/any.h> #include <validation.h> #include <any> diff --git a/src/rpc/txoutproof.cpp b/src/rpc/txoutproof.cpp index 24b5d04115..d74959cecc 100644 --- a/src/rpc/txoutproof.cpp +++ b/src/rpc/txoutproof.cpp @@ -18,7 +18,6 @@ #include <validation.h> using node::GetTransaction; -using node::ReadBlockFromDisk; static RPCHelpMan gettxoutproof() { @@ -85,7 +84,7 @@ static RPCHelpMan gettxoutproof() } if (pblockindex == nullptr) { - const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, /*mempool=*/nullptr, *setTxids.begin(), chainman.GetConsensus(), hashBlock); + const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, /*mempool=*/nullptr, *setTxids.begin(), hashBlock, chainman.m_blockman); if (!tx || hashBlock.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); } @@ -98,7 +97,7 @@ static RPCHelpMan gettxoutproof() } CBlock block; - if (!ReadBlockFromDisk(block, pblockindex, chainman.GetConsensus())) { + if (!chainman.m_blockman.ReadBlockFromDisk(block, *pblockindex)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); } diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 1f3f37d0a0..19e14f88df 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -37,7 +37,7 @@ void RPCTypeCheckObj(const UniValue& o, bool fStrict) { for (const auto& t : typesExpected) { - const UniValue& v = find_value(o, t.first); + const UniValue& v = o.find_value(t.first); if (!fAllowNull && v.isNull()) throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing %s", t.first)); @@ -81,7 +81,7 @@ uint256 ParseHashV(const UniValue& v, std::string strName) } uint256 ParseHashO(const UniValue& o, std::string strKey) { - return ParseHashV(find_value(o, strKey), strKey); + return ParseHashV(o.find_value(strKey), strKey); } std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName) { @@ -94,7 +94,7 @@ std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName) } std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) { - return ParseHexV(find_value(o, strKey), strKey); + return ParseHexV(o.find_value(strKey), strKey); } namespace { @@ -389,7 +389,8 @@ struct Sections { case RPCArg::Type::NUM: case RPCArg::Type::AMOUNT: case RPCArg::Type::RANGE: - case RPCArg::Type::BOOL: { + case RPCArg::Type::BOOL: + case RPCArg::Type::OBJ_NAMED_PARAMS: { if (is_top_level_arg) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; if (arg.m_opts.type_str.size() != 0 && push_name) { @@ -485,12 +486,32 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP m_results{std::move(results)}, m_examples{std::move(examples)} { - std::set<std::string> named_args; + // Map of parameter names and types just used to check whether the names are + // unique. Parameter names always need to be unique, with the exception that + // there can be pairs of POSITIONAL and NAMED parameters with the same name. + enum ParamType { POSITIONAL = 1, NAMED = 2, NAMED_ONLY = 4 }; + std::map<std::string, int> param_names; + for (const auto& arg : m_args) { std::vector<std::string> names = SplitString(arg.m_names, '|'); // Should have unique named arguments for (const std::string& name : names) { - CHECK_NONFATAL(named_args.insert(name).second); + auto& param_type = param_names[name]; + CHECK_NONFATAL(!(param_type & POSITIONAL)); + CHECK_NONFATAL(!(param_type & NAMED_ONLY)); + param_type |= POSITIONAL; + } + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& inner : arg.m_inner) { + std::vector<std::string> inner_names = SplitString(inner.m_names, '|'); + for (const std::string& inner_name : inner_names) { + auto& param_type = param_names[inner_name]; + CHECK_NONFATAL(!(param_type & POSITIONAL) || inner.m_opts.also_positional); + CHECK_NONFATAL(!(param_type & NAMED)); + CHECK_NONFATAL(!(param_type & NAMED_ONLY)); + param_type |= inner.m_opts.also_positional ? NAMED : NAMED_ONLY; + } + } } // Default value type should match argument type only when defined if (arg.m_fallback.index() == 2) { @@ -605,12 +626,17 @@ bool RPCHelpMan::IsValidNumArgs(size_t num_args) const return num_required_args <= num_args && num_args <= m_args.size(); } -std::vector<std::string> RPCHelpMan::GetArgNames() const +std::vector<std::pair<std::string, bool>> RPCHelpMan::GetArgNames() const { - std::vector<std::string> ret; + std::vector<std::pair<std::string, bool>> ret; ret.reserve(m_args.size()); for (const auto& arg : m_args) { - ret.emplace_back(arg.m_names); + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& inner : arg.m_inner) { + ret.emplace_back(inner.m_names, /*named_only=*/true); + } + } + ret.emplace_back(arg.m_names, /*named_only=*/false); } return ret; } @@ -642,20 +668,31 @@ std::string RPCHelpMan::ToString() const // Arguments Sections sections; + Sections named_only_sections; for (size_t i{0}; i < m_args.size(); ++i) { const auto& arg = m_args.at(i); if (arg.m_opts.hidden) break; // Any arg that follows is also hidden - if (i == 0) ret += "\nArguments:\n"; - // Push named argument name and description sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString(/*is_named_arg=*/true)); sections.m_max_pad = std::max(sections.m_max_pad, sections.m_sections.back().m_left.size()); // Recursively push nested args sections.Push(arg); + + // Push named-only argument sections + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& arg_inner : arg.m_inner) { + named_only_sections.PushSection({arg_inner.GetFirstName(), arg_inner.ToDescriptionString(/*is_named_arg=*/true)}); + named_only_sections.Push(arg_inner); + } + } } + + if (!sections.m_sections.empty()) ret += "\nArguments:\n"; ret += sections.ToString(); + if (!named_only_sections.m_sections.empty()) ret += "\nNamed Arguments:\n"; + ret += named_only_sections.ToString(); // Result ret += m_results.ToDescriptionString(); @@ -669,17 +706,30 @@ std::string RPCHelpMan::ToString() const UniValue RPCHelpMan::GetArgMap() const { UniValue arr{UniValue::VARR}; + + auto push_back_arg_info = [&arr](const std::string& rpc_name, int pos, const std::string& arg_name, const RPCArg::Type& type) { + UniValue map{UniValue::VARR}; + map.push_back(rpc_name); + map.push_back(pos); + map.push_back(arg_name); + map.push_back(type == RPCArg::Type::STR || + type == RPCArg::Type::STR_HEX); + arr.push_back(map); + }; + for (int i{0}; i < int(m_args.size()); ++i) { const auto& arg = m_args.at(i); std::vector<std::string> arg_names = SplitString(arg.m_names, '|'); for (const auto& arg_name : arg_names) { - UniValue map{UniValue::VARR}; - map.push_back(m_name); - map.push_back(i); - map.push_back(arg_name); - map.push_back(arg.m_type == RPCArg::Type::STR || - arg.m_type == RPCArg::Type::STR_HEX); - arr.push_back(map); + push_back_arg_info(m_name, i, arg_name, arg.m_type); + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& inner : arg.m_inner) { + std::vector<std::string> inner_names = SplitString(inner.m_names, '|'); + for (const std::string& inner_name : inner_names) { + push_back_arg_info(m_name, i, inner_name, inner.m_type); + } + } + } } } return arr; @@ -708,6 +758,7 @@ static std::optional<UniValue::VType> ExpectedType(RPCArg::Type type) return UniValue::VBOOL; } case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { return UniValue::VOBJ; } @@ -781,6 +832,7 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const break; } case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { ret += "json object"; break; @@ -809,6 +861,7 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const } // no default case, so the compiler can warn about missing cases } ret += ")"; + if (m_type == Type::OBJ_NAMED_PARAMS) ret += " Options object that can be used to pass named arguments, listed below."; ret += m_description.empty() ? "" : " " + m_description; return ret; } @@ -1054,6 +1107,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const } return res + "...]"; case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: // Currently unused, so avoid writing dead code NONFATAL_UNREACHABLE(); @@ -1077,6 +1131,7 @@ std::string RPCArg::ToString(const bool oneline) const return GetFirstName(); } case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { const std::string res = Join(m_inner, ",", [&](const RPCArg& i) { return i.ToStringObj(oneline); }); if (m_type == Type::OBJ) { @@ -1126,17 +1181,17 @@ std::pair<int64_t, int64_t> ParseDescriptorRange(const UniValue& value) return {low, high}; } -std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider) +std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider, const bool expand_priv) { std::string desc_str; std::pair<int64_t, int64_t> range = {0, 1000}; if (scanobject.isStr()) { desc_str = scanobject.get_str(); } else if (scanobject.isObject()) { - UniValue desc_uni = find_value(scanobject, "desc"); + const UniValue& desc_uni{scanobject.find_value("desc")}; if (desc_uni.isNull()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Descriptor needs to be provided in scan object"); desc_str = desc_uni.get_str(); - UniValue range_uni = find_value(scanobject, "range"); + const UniValue& range_uni{scanobject.find_value("range")}; if (!range_uni.isNull()) { range = ParseDescriptorRange(range_uni); } @@ -1159,6 +1214,9 @@ std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, Fl if (!desc->Expand(i, provider, scripts, provider)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys: '%s'", desc_str)); } + if (expand_priv) { + desc->ExpandPrivate(/*pos=*/i, provider, /*out=*/provider); + } std::move(scripts.begin(), scripts.end(), std::back_inserter(ret)); } return ret; diff --git a/src/rpc/util.h b/src/rpc/util.h index bb5c30a2f4..4cba5a9818 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -110,7 +110,7 @@ UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_s std::pair<int64_t, int64_t> ParseDescriptorRange(const UniValue& value); /** Evaluate a descriptor given as a string, or as a {"desc":...,"range":...} object, with default range of 1000. */ -std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider); +std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider, const bool expand_priv = false); /** Returns, given services flags, a list of humanly readable (known) network services */ UniValue GetServicesNames(ServiceFlags services); @@ -130,6 +130,15 @@ struct RPCArgOptions { std::string oneline_description{}; //!< Should be empty unless it is supposed to override the auto-generated summary line std::vector<std::string> type_str{}; //!< Should be empty unless it is supposed to override the auto-generated type strings. Vector length is either 0 or 2, m_opts.type_str.at(0) will override the type of the value in a key-value pair, m_opts.type_str.at(1) will override the type in the argument description. bool hidden{false}; //!< For testing only + bool also_positional{false}; //!< If set allows a named-parameter field in an OBJ_NAMED_PARAM options object + //!< to have the same name as a top-level parameter. By default the RPC + //!< framework disallows this, because if an RPC request passes the value by + //!< name, it is assigned to top-level parameter position, not to the options + //!< position, defeating the purpose of using OBJ_NAMED_PARAMS instead OBJ for + //!< that option. But sometimes it makes sense to allow less-commonly used + //!< options to be passed by name only, and more commonly used options to be + //!< passed by name or position, so the RPC framework allows this as long as + //!< methods set the also_positional flag and read values from both positions. }; struct RPCArg { @@ -139,6 +148,13 @@ struct RPCArg { STR, NUM, BOOL, + OBJ_NAMED_PARAMS, //!< Special type that behaves almost exactly like + //!< OBJ, defining an options object with a list of + //!< pre-defined keys. The only difference between OBJ + //!< and OBJ_NAMED_PARAMS is that OBJ_NAMED_PARMS + //!< also allows the keys to be passed as top-level + //!< named parameters, as a more convenient way to pass + //!< options to the RPC method without nesting them. OBJ_USER_KEYS, //!< Special type where the user must set the keys e.g. to define multiple addresses; as opposed to e.g. an options object where the keys are predefined AMOUNT, //!< Special type representing a floating point amount (can be either NUM or STR) STR_HEX, //!< Special type that is a STR with only hex chars @@ -183,7 +199,7 @@ struct RPCArg { m_description{std::move(description)}, m_opts{std::move(opts)} { - CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ && type != Type::OBJ_USER_KEYS); + CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ && type != Type::OBJ_NAMED_PARAMS && type != Type::OBJ_USER_KEYS); } RPCArg( @@ -200,7 +216,7 @@ struct RPCArg { m_description{std::move(description)}, m_opts{std::move(opts)} { - CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ || type == Type::OBJ_USER_KEYS); + CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ || type == Type::OBJ_NAMED_PARAMS || type == Type::OBJ_USER_KEYS); } bool IsOptional() const; @@ -369,7 +385,8 @@ public: UniValue GetArgMap() const; /** If the supplied number of args is neither too small nor too high */ bool IsValidNumArgs(size_t num_args) const; - std::vector<std::string> GetArgNames() const; + //! Return list of arguments and whether they are named-only. + std::vector<std::pair<std::string, bool>> GetArgNames() const; const std::string m_name; diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 0951504ee0..b8ade1684a 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -197,7 +197,9 @@ public: /** Get the descriptor string form including private data (if available in arg). */ virtual bool ToPrivateString(const SigningProvider& arg, std::string& out) const = 0; - /** Get the descriptor string form with the xpub at the last hardened derivation */ + /** Get the descriptor string form with the xpub at the last hardened derivation, + * and always use h for hardened derivation. + */ virtual bool ToNormalizedString(const SigningProvider& arg, std::string& out, const DescriptorCache* cache = nullptr) const = 0; /** Derive a private key, if private data is available in arg. */ @@ -208,14 +210,15 @@ class OriginPubkeyProvider final : public PubkeyProvider { KeyOriginInfo m_origin; std::unique_ptr<PubkeyProvider> m_provider; + bool m_apostrophe; - std::string OriginString() const + std::string OriginString(bool normalized=false) const { - return HexStr(m_origin.fingerprint) + FormatHDKeypath(m_origin.path); + return HexStr(m_origin.fingerprint) + FormatHDKeypath(m_origin.path, /*apostrophe=*/!normalized && m_apostrophe); } public: - OriginPubkeyProvider(uint32_t exp_index, KeyOriginInfo info, std::unique_ptr<PubkeyProvider> provider) : PubkeyProvider(exp_index), m_origin(std::move(info)), m_provider(std::move(provider)) {} + OriginPubkeyProvider(uint32_t exp_index, KeyOriginInfo info, std::unique_ptr<PubkeyProvider> provider, bool apostrophe) : PubkeyProvider(exp_index), m_origin(std::move(info)), m_provider(std::move(provider)), m_apostrophe(apostrophe) {} bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key, KeyOriginInfo& info, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) const override { if (!m_provider->GetPubKey(pos, arg, key, info, read_cache, write_cache)) return false; @@ -242,9 +245,9 @@ public: // and append that to our own origin string. if (sub[0] == '[') { sub = sub.substr(9); - ret = "[" + OriginString() + std::move(sub); + ret = "[" + OriginString(/*normalized=*/true) + std::move(sub); } else { - ret = "[" + OriginString() + "]" + std::move(sub); + ret = "[" + OriginString(/*normalized=*/true) + "]" + std::move(sub); } return true; } @@ -312,6 +315,8 @@ class BIP32PubkeyProvider final : public PubkeyProvider CExtPubKey m_root_extkey; KeyPath m_path; DeriveType m_derive; + // Whether ' or h is used in harded derivation + bool m_apostrophe; bool GetExtKey(const SigningProvider& arg, CExtKey& ret) const { @@ -348,7 +353,7 @@ class BIP32PubkeyProvider final : public PubkeyProvider } public: - BIP32PubkeyProvider(uint32_t exp_index, const CExtPubKey& extkey, KeyPath path, DeriveType derive) : PubkeyProvider(exp_index), m_root_extkey(extkey), m_path(std::move(path)), m_derive(derive) {} + BIP32PubkeyProvider(uint32_t exp_index, const CExtPubKey& extkey, KeyPath path, DeriveType derive, bool apostrophe) : PubkeyProvider(exp_index), m_root_extkey(extkey), m_path(std::move(path)), m_derive(derive), m_apostrophe(apostrophe) {} bool IsRange() const override { return m_derive != DeriveType::NO; } size_t GetSize() const override { return 33; } bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key_out, KeyOriginInfo& final_info_out, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) const override @@ -416,31 +421,36 @@ public: return true; } - std::string ToString() const override + std::string ToString(bool normalized) const { - std::string ret = EncodeExtPubKey(m_root_extkey) + FormatHDKeypath(m_path); + const bool use_apostrophe = !normalized && m_apostrophe; + std::string ret = EncodeExtPubKey(m_root_extkey) + FormatHDKeypath(m_path, /*apostrophe=*/use_apostrophe); if (IsRange()) { ret += "/*"; - if (m_derive == DeriveType::HARDENED) ret += '\''; + if (m_derive == DeriveType::HARDENED) ret += use_apostrophe ? '\'' : 'h'; } return ret; } + std::string ToString() const override + { + return ToString(/*normalized=*/false); + } bool ToPrivateString(const SigningProvider& arg, std::string& out) const override { CExtKey key; if (!GetExtKey(arg, key)) return false; - out = EncodeExtKey(key) + FormatHDKeypath(m_path); + out = EncodeExtKey(key) + FormatHDKeypath(m_path, /*apostrophe=*/m_apostrophe); if (IsRange()) { out += "/*"; - if (m_derive == DeriveType::HARDENED) out += '\''; + if (m_derive == DeriveType::HARDENED) out += m_apostrophe ? '\'' : 'h'; } return true; } bool ToNormalizedString(const SigningProvider& arg, std::string& out, const DescriptorCache* cache) const override { - // For hardened derivation type, just return the typical string, nothing to normalize if (m_derive == DeriveType::HARDENED) { - out = ToString(); + out = ToString(/*normalized=*/true); + return true; } // Step backwards to find the last hardened step in the path @@ -1049,15 +1059,27 @@ enum class ParseScriptContext { P2TR, //!< Inside tr() (either internal key, or BIP342 script leaf) }; -/** Parse a key path, being passed a split list of elements (the first element is ignored). */ -[[nodiscard]] bool ParseKeyPath(const std::vector<Span<const char>>& split, KeyPath& out, std::string& error) +/** + * Parse a key path, being passed a split list of elements (the first element is ignored). + * + * @param[in] split BIP32 path string, using either ' or h for hardened derivation + * @param[out] out the key path + * @param[out] apostrophe only updated if hardened derivation is found + * @param[out] error parsing error message + * @returns false if parsing failed + **/ +[[nodiscard]] bool ParseKeyPath(const std::vector<Span<const char>>& split, KeyPath& out, bool& apostrophe, std::string& error) { for (size_t i = 1; i < split.size(); ++i) { Span<const char> elem = split[i]; bool hardened = false; - if (elem.size() > 0 && (elem[elem.size() - 1] == '\'' || elem[elem.size() - 1] == 'h')) { - elem = elem.first(elem.size() - 1); - hardened = true; + if (elem.size() > 0) { + const char last = elem[elem.size() - 1]; + if (last == '\'' || last == 'h') { + elem = elem.first(elem.size() - 1); + hardened = true; + apostrophe = last == '\''; + } } uint32_t p; if (!ParseUInt32(std::string(elem.begin(), elem.end()), &p)) { @@ -1073,7 +1095,7 @@ enum class ParseScriptContext { } /** Parse a public key that excludes origin information. */ -std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) +std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, bool& apostrophe, std::string& error) { using namespace spanparsing; @@ -1130,15 +1152,16 @@ std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const S split.pop_back(); type = DeriveType::UNHARDENED; } else if (split.back() == Span{"*'"}.first(2) || split.back() == Span{"*h"}.first(2)) { + apostrophe = split.back() == Span{"*'"}.first(2); split.pop_back(); type = DeriveType::HARDENED; } - if (!ParseKeyPath(split, path, error)) return nullptr; + if (!ParseKeyPath(split, path, apostrophe, error)) return nullptr; if (extkey.key.IsValid()) { extpubkey = extkey.Neuter(); out.keys.emplace(extpubkey.pubkey.GetID(), extkey.key); } - return std::make_unique<BIP32PubkeyProvider>(key_exp_index, extpubkey, std::move(path), type); + return std::make_unique<BIP32PubkeyProvider>(key_exp_index, extpubkey, std::move(path), type, apostrophe); } /** Parse a public key including origin information (if enabled). */ @@ -1151,7 +1174,11 @@ std::unique_ptr<PubkeyProvider> ParsePubkey(uint32_t key_exp_index, const Span<c error = "Multiple ']' characters found for a single pubkey"; return nullptr; } - if (origin_split.size() == 1) return ParsePubkeyInner(key_exp_index, origin_split[0], ctx, out, error); + // This is set if either the origin or path suffix contains a hardened derivation. + bool apostrophe = false; + if (origin_split.size() == 1) { + return ParsePubkeyInner(key_exp_index, origin_split[0], ctx, out, apostrophe, error); + } if (origin_split[0].empty() || origin_split[0][0] != '[') { error = strprintf("Key origin start '[ character expected but not found, got '%c' instead", origin_split[0].empty() ? /** empty, implies split char */ ']' : origin_split[0][0]); @@ -1172,10 +1199,10 @@ std::unique_ptr<PubkeyProvider> ParsePubkey(uint32_t key_exp_index, const Span<c static_assert(sizeof(info.fingerprint) == 4, "Fingerprint must be 4 bytes"); assert(fpr_bytes.size() == 4); std::copy(fpr_bytes.begin(), fpr_bytes.end(), info.fingerprint); - if (!ParseKeyPath(slash_split, info.path, error)) return nullptr; - auto provider = ParsePubkeyInner(key_exp_index, origin_split[1], ctx, out, error); + if (!ParseKeyPath(slash_split, info.path, apostrophe, error)) return nullptr; + auto provider = ParsePubkeyInner(key_exp_index, origin_split[1], ctx, out, apostrophe, error); if (!provider) return nullptr; - return std::make_unique<OriginPubkeyProvider>(key_exp_index, std::move(info), std::move(provider)); + return std::make_unique<OriginPubkeyProvider>(key_exp_index, std::move(info), std::move(provider), apostrophe); } std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptContext, const SigningProvider& provider) @@ -1183,7 +1210,7 @@ std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptCo std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, false); KeyOriginInfo info; if (provider.GetKeyOrigin(pubkey.GetID(), info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); + return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider), /*apostrophe=*/false); } return key_provider; } @@ -1196,7 +1223,7 @@ std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseS std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, true); KeyOriginInfo info; if (provider.GetKeyOriginByXOnly(xkey, info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); + return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider), /*apostrophe=*/false); } return key_provider; } diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index e52e8cd309..7c6c282cc4 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -5,11 +5,11 @@ #include <script/sigcache.h> +#include <common/system.h> #include <logging.h> #include <pubkey.h> #include <random.h> #include <uint256.h> -#include <util/system.h> #include <cuckoocache.h> diff --git a/src/shutdown.cpp b/src/shutdown.cpp index 2fffc0663c..d70017d734 100644 --- a/src/shutdown.cpp +++ b/src/shutdown.cpp @@ -11,6 +11,7 @@ #include <logging.h> #include <node/interface_ui.h> +#include <util/check.h> #include <util/tokenpipe.h> #include <warnings.h> @@ -20,6 +21,8 @@ #include <condition_variable> #endif +static std::atomic<int>* g_exit_status{nullptr}; + bool AbortNode(const std::string& strMessage, bilingual_str user_message) { SetMiscWarning(Untranslated(strMessage)); @@ -28,6 +31,7 @@ bool AbortNode(const std::string& strMessage, bilingual_str user_message) user_message = _("A fatal internal error occurred, see debug.log for details"); } InitError(user_message); + Assert(g_exit_status)->store(EXIT_FAILURE); StartShutdown(); return false; } @@ -44,8 +48,9 @@ static TokenPipeEnd g_shutdown_r; static TokenPipeEnd g_shutdown_w; #endif -bool InitShutdownState() +bool InitShutdownState(std::atomic<int>& exit_status) { + g_exit_status = &exit_status; #ifndef WIN32 std::optional<TokenPipe> pipe = TokenPipe::Make(); if (!pipe) return false; diff --git a/src/shutdown.h b/src/shutdown.h index 07a8315788..c119bee96f 100644 --- a/src/shutdown.h +++ b/src/shutdown.h @@ -8,13 +8,15 @@ #include <util/translation.h> // For bilingual_str +#include <atomic> + /** Abort with a message */ bool AbortNode(const std::string& strMessage, bilingual_str user_message = bilingual_str{}); /** Initialize shutdown state. This must be called before using either StartShutdown(), * AbortShutdown() or WaitForShutdown(). Calling ShutdownRequested() is always safe. */ -bool InitShutdownState(); +bool InitShutdownState(std::atomic<int>& exit_status); /** Request shutdown of the application. */ void StartShutdown(); diff --git a/src/signet.cpp b/src/signet.cpp index b76b1e342f..b73d82bb2e 100644 --- a/src/signet.cpp +++ b/src/signet.cpp @@ -8,6 +8,7 @@ #include <cstdint> #include <vector> +#include <common/system.h> #include <consensus/merkle.h> #include <consensus/params.h> #include <consensus/validation.h> @@ -22,7 +23,6 @@ #include <streams.h> #include <uint256.h> #include <util/strencodings.h> -#include <util/system.h> static constexpr uint8_t SIGNET_HEADER[4] = {0xec, 0xc7, 0xda, 0xa2}; diff --git a/src/streams.h b/src/streams.h index 8788343809..03df20b5db 100644 --- a/src/streams.h +++ b/src/streams.h @@ -293,14 +293,6 @@ public: vch.insert(vch.end(), src.begin(), src.end()); } - template<typename Stream> - void Serialize(Stream& s) const - { - // Special case: stream << stream concatenates like stream += stream - if (!vch.empty()) - s.write(MakeByteSpan(vch)); - } - template<typename T> DataStream& operator<<(const T& obj) { @@ -756,15 +748,25 @@ public: } //! search for a given byte in the stream, and remain positioned on it - void FindByte(uint8_t ch) + void FindByte(std::byte byte) { + // For best performance, avoid mod operation within the loop. + size_t buf_offset{size_t(m_read_pos % uint64_t(vchBuf.size()))}; while (true) { - if (m_read_pos == nSrcPos) + if (m_read_pos == nSrcPos) { + // No more bytes available; read from the file into the buffer, + // setting nSrcPos to one beyond the end of the new data. + // Throws exception if end-of-file reached. Fill(); - if (vchBuf[m_read_pos % vchBuf.size()] == std::byte{ch}) { - break; } - m_read_pos++; + const size_t len{std::min<size_t>(vchBuf.size() - buf_offset, nSrcPos - m_read_pos)}; + const auto it_start{vchBuf.begin() + buf_offset}; + const auto it_find{std::find(it_start, it_start + len, byte)}; + const size_t inc{size_t(std::distance(it_start, it_find))}; + m_read_pos += inc; + if (inc < len) break; + buf_offset += inc; + if (buf_offset >= vchBuf.size()) buf_offset = 0; } } }; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 54d923e4a4..328e7f81a0 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -33,16 +33,16 @@ static int32_t GetCheckRatio(const NodeContext& node_ctx) static CNetAddr ResolveIP(const std::string& ip) { - CNetAddr addr; - BOOST_CHECK_MESSAGE(LookupHost(ip, addr, false), strprintf("failed to resolve: %s", ip)); - return addr; + const std::optional<CNetAddr> addr{LookupHost(ip, false)}; + BOOST_CHECK_MESSAGE(addr.has_value(), strprintf("failed to resolve: %s", ip)); + return addr.value_or(CNetAddr{}); } static CService ResolveService(const std::string& ip, uint16_t port = 0) { - CService serv; - BOOST_CHECK_MESSAGE(Lookup(ip, serv, port, false), strprintf("failed to resolve: %s:%i", ip, port)); - return serv; + const std::optional<CService> serv{Lookup(ip, port, false)}; + BOOST_CHECK_MESSAGE(serv.has_value(), strprintf("failed to resolve: %s:%i", ip, port)); + return serv.value_or(CService{}); } @@ -948,18 +948,23 @@ BOOST_AUTO_TEST_CASE(load_addrman) { AddrMan addrman{EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)}; - CService addr1, addr2, addr3; - BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); - BOOST_CHECK(Lookup("250.7.2.2", addr2, 9999, false)); - BOOST_CHECK(Lookup("250.7.3.3", addr3, 9999, false)); - BOOST_CHECK(Lookup("250.7.3.3"s, addr3, 9999, false)); - BOOST_CHECK(!Lookup("250.7.3.3\0example.com"s, addr3, 9999, false)); + std::optional<CService> addr1, addr2, addr3, addr4; + addr1 = Lookup("250.7.1.1", 8333, false); + BOOST_CHECK(addr1.has_value()); + addr2 = Lookup("250.7.2.2", 9999, false); + BOOST_CHECK(addr2.has_value()); + addr3 = Lookup("250.7.3.3", 9999, false); + BOOST_CHECK(addr3.has_value()); + addr3 = Lookup("250.7.3.3"s, 9999, false); + BOOST_CHECK(addr3.has_value()); + addr4 = Lookup("250.7.3.3\0example.com"s, 9999, false); + BOOST_CHECK(!addr4.has_value()); // Add three addresses to new table. - CService source; - BOOST_CHECK(Lookup("252.5.1.1", source, 8333, false)); - std::vector<CAddress> addresses{CAddress(addr1, NODE_NONE), CAddress(addr2, NODE_NONE), CAddress(addr3, NODE_NONE)}; - BOOST_CHECK(addrman.Add(addresses, source)); + const std::optional<CService> source{Lookup("252.5.1.1", 8333, false)}; + BOOST_CHECK(source.has_value()); + std::vector<CAddress> addresses{CAddress(addr1.value(), NODE_NONE), CAddress(addr2.value(), NODE_NONE), CAddress(addr3.value(), NODE_NONE)}; + BOOST_CHECK(addrman.Add(addresses, source.value())); BOOST_CHECK(addrman.Size() == 3); // Test that the de-serialization does not throw an exception. @@ -1004,12 +1009,12 @@ static CDataStream MakeCorruptPeersDat() int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); s << nUBuckets; - CService serv; - BOOST_CHECK(Lookup("252.1.1.1", serv, 7777, false)); - CAddress addr = CAddress(serv, NODE_NONE); - CNetAddr resolved; - BOOST_CHECK(LookupHost("252.2.2.2", resolved, false)); - AddrInfo info = AddrInfo(addr, resolved); + const std::optional<CService> serv{Lookup("252.1.1.1", 7777, false)}; + BOOST_REQUIRE(serv.has_value()); + CAddress addr = CAddress(serv.value(), NODE_NONE); + std::optional<CNetAddr> resolved{LookupHost("252.2.2.2", false)}; + BOOST_REQUIRE(resolved.has_value()); + AddrInfo info = AddrInfo(addr, resolved.value()); s << info; return s; diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index f74e50a890..8c0af6f26f 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <support/lockedpool.h> -#include <util/system.h> #include <limits> #include <memory> diff --git a/src/test/argsman_tests.cpp b/src/test/argsman_tests.cpp index 6a0925f0bb..0b789e7f5c 100644 --- a/src/test/argsman_tests.cpp +++ b/src/test/argsman_tests.cpp @@ -8,6 +8,7 @@ #include <test/util/setup_common.h> #include <test/util/str.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/fs.h> #include <util/strencodings.h> @@ -84,7 +85,7 @@ class CheckValueTest : public TestChain100Setup { public: struct Expect { - util::SettingsValue setting; + common::SettingsValue setting; bool default_string = false; bool default_int = false; bool default_bool = false; @@ -94,7 +95,7 @@ public: std::optional<std::vector<std::string>> list_value; const char* error = nullptr; - explicit Expect(util::SettingsValue s) : setting(std::move(s)) {} + explicit Expect(common::SettingsValue s) : setting(std::move(s)) {} Expect& DefaultString() { default_string = true; return *this; } Expect& DefaultInt() { default_int = true; return *this; } Expect& DefaultBool() { default_bool = true; return *this; } @@ -254,7 +255,7 @@ BOOST_AUTO_TEST_CASE(util_ParseInvalidParameters) BOOST_CHECK(!test.ParseParameters(2, (char**)argv, error)); BOOST_CHECK_EQUAL(error, "Invalid parameter -unregistered"); - // Make sure registered parameters prefixed with a chain name trigger errors. + // Make sure registered parameters prefixed with a chain type trigger errors. // (Previously, they were accepted and ignored.) argv[1] = "-test.registered"; BOOST_CHECK(!test.ParseParameters(2, (char**)argv, error)); @@ -580,7 +581,7 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) test_args.SetNetworkOnlyArg("-ccc"); test_args.SetNetworkOnlyArg("-h"); - test_args.SelectConfigNetwork(CBaseChainParams::MAIN); + test_args.SelectConfigNetwork(ChainTypeToString(ChainType::MAIN)); BOOST_CHECK(test_args.GetArg("-d", "xxx") == "e"); BOOST_CHECK(test_args.GetArgs("-ccc").size() == 2); BOOST_CHECK(test_args.GetArg("-h", "xxx") == "0"); @@ -637,7 +638,7 @@ BOOST_AUTO_TEST_CASE(util_GetArg) BOOST_CHECK_EQUAL(testArgs.GetArg("pritest4", "default"), "b"); } -BOOST_AUTO_TEST_CASE(util_GetChainName) +BOOST_AUTO_TEST_CASE(util_GetChainTypeString) { TestArgsManager test_args; const auto testnet = std::make_pair("-testnet", ArgsManager::ALLOW_ANY); @@ -655,39 +656,39 @@ BOOST_AUTO_TEST_CASE(util_GetChainName) std::string error; BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "main"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "main"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "regtest"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "regtest"); BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_test_no_reg, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_test_no_reg, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); // check setting the network to test (and thus making // [test] regtest=1 potentially relevant) doesn't break things @@ -695,23 +696,23 @@ BOOST_AUTO_TEST_CASE(util_GetChainName) BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_test_no_reg, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); } // Test different ways settings can be merged, and verify results. This test can @@ -755,8 +756,8 @@ struct ArgsMergeTestingSetup : public BasicTestingSetup { ForEachNoDup(conf_actions, SET, SECTION_NEGATE, [&] { for (bool soft_set : {false, true}) { for (bool force_set : {false, true}) { - for (const std::string& section : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET}) { - for (const std::string& network : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET}) { + for (const std::string& section : {ChainTypeToString(ChainType::MAIN), ChainTypeToString(ChainType::TESTNET), ChainTypeToString(ChainType::SIGNET)}) { + for (const std::string& network : {ChainTypeToString(ChainType::MAIN), ChainTypeToString(ChainType::TESTNET), ChainTypeToString(ChainType::SIGNET)}) { for (bool net_specific : {false, true}) { fn(arg_actions, conf_actions, soft_set, force_set, section, network, net_specific); } @@ -913,7 +914,7 @@ BOOST_FIXTURE_TEST_CASE(util_ArgsMerge, ArgsMergeTestingSetup) BOOST_CHECK_EQUAL(out_sha_hex, "d1e436c1cd510d0ec44d5205d4b4e3bee6387d316e0075c58206cb16603f3d82"); } -// Similar test as above, but for ArgsManager::GetChainName function. +// Similar test as above, but for ArgsManager::GetChainTypeString function. struct ChainMergeTestingSetup : public BasicTestingSetup { static constexpr int MAX_ACTIONS = 2; @@ -982,7 +983,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) desc += " || "; try { - desc += parser.GetChainName(); + desc += parser.GetChainTypeString(); } catch (const std::runtime_error& e) { desc += "error: "; desc += e.what(); @@ -1021,14 +1022,14 @@ BOOST_AUTO_TEST_CASE(util_ReadWriteSettings) // Test writing setting. TestArgsManager args1; args1.ForceSetArg("-datadir", fs::PathToString(m_path_root)); - args1.LockSettings([&](util::Settings& settings) { settings.rw_settings["name"] = "value"; }); + args1.LockSettings([&](common::Settings& settings) { settings.rw_settings["name"] = "value"; }); args1.WriteSettingsFile(); // Test reading setting. TestArgsManager args2; args2.ForceSetArg("-datadir", fs::PathToString(m_path_root)); args2.ReadSettingsFile(); - args2.LockSettings([&](util::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); + args2.LockSettings([&](common::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); // Test error logging, and remove previously written setting. { diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index a572bb02b9..1ff5d6cf59 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -19,6 +19,7 @@ #include <boost/test/unit_test.hpp> using node::BlockAssembler; +using node::BlockManager; using node::CBlockTemplate; BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) @@ -29,10 +30,10 @@ struct BuildChainTestingSetup : public TestChain100Setup { }; static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index, - uint256& last_header) + uint256& last_header, const BlockManager& blockman) { BlockFilter expected_filter; - if (!ComputeFilter(filter_index.GetFilterType(), block_index, expected_filter)) { + if (!ComputeFilter(filter_index.GetFilterType(), *block_index, expected_filter, blockman)) { BOOST_ERROR("ComputeFilter failed on block " << block_index->nHeight); return false; } @@ -141,10 +142,10 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) BOOST_REQUIRE(filter_index.Start()); // Allow filter index to catch up with the block index. - constexpr int64_t timeout_ms = 10 * 1000; - int64_t time_start = GetTimeMillis(); + constexpr auto timeout{10s}; + const auto time_start{SteadyClock::now()}; while (!filter_index.BlockUntilSyncedToCurrentChain()) { - BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis()); + BOOST_REQUIRE(time_start + timeout > SteadyClock::now()); UninterruptibleSleep(std::chrono::milliseconds{100}); } @@ -155,7 +156,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) for (block_index = m_node.chainman->ActiveChain().Genesis(); block_index != nullptr; block_index = m_node.chainman->ActiveChain().Next(block_index)) { - CheckFilterLookups(filter_index, block_index, last_header); + CheckFilterLookups(filter_index, block_index, last_header, m_node.chainman->m_blockman); } } @@ -189,7 +190,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainA_last_header); + CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman); } // Reorg to chain B. @@ -207,7 +208,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainB_last_header); + CheckFilterLookups(filter_index, block_index, chainB_last_header, m_node.chainman->m_blockman); } // Check that filters for stale blocks on A can be retrieved. @@ -221,7 +222,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainA_last_header); + CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman); } // Reorg back to chain A. @@ -241,14 +242,14 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) block_index = m_node.chainman->m_blockman.LookupBlockIndex(chainA[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainA_last_header); + CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman); { LOCK(cs_main); block_index = m_node.chainman->m_blockman.LookupBlockIndex(chainB[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainB_last_header); + CheckFilterLookups(filter_index, block_index, chainB_last_header, m_node.chainman->m_blockman); } // Test lookups for a range of filters/hashes. diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index 2118f476cd..f094766886 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -5,6 +5,7 @@ #include <chainparams.h> #include <node/blockstorage.h> #include <node/context.h> +#include <util/chaintype.h> #include <validation.h> #include <boost/test/unit_test.hpp> @@ -13,31 +14,34 @@ using node::BlockManager; using node::BLOCK_SERIALIZATION_HEADER_SIZE; using node::MAX_BLOCKFILE_SIZE; -using node::OpenBlockFile; // use BasicTestingSetup here for the data directory configuration, setup, and cleanup BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) { - const auto params {CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN)}; - BlockManager blockman{{}}; + const auto params {CreateChainParams(ArgsManager{}, ChainType::MAIN)}; + const BlockManager::Options blockman_opts{ + .chainparams = *params, + .blocks_dir = m_args.GetBlocksDirPath(), + }; + BlockManager blockman{blockman_opts}; CChain chain {}; // simulate adding a genesis block normally - BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // simulate what happens during reindex // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file // the block is found at offset 8 because there is an 8 byte serialization header // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file. FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; - BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // now simulate what happens after reindex for the first new block processed // the actual block contents don't matter, just that it's a block. // verify that the write position is at offset 0x12d. // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 - FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, *params, nullptr)}; + FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, nullptr)}; BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); } @@ -63,13 +67,13 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_scan_unlink_already_pruned_files, TestChain // Check that the file is not unlinked after ScanAndUnlinkAlreadyPrunedFiles // if m_have_pruned is not yet set WITH_LOCK(chainman->GetMutex(), blockman.ScanAndUnlinkAlreadyPrunedFiles()); - BOOST_CHECK(!AutoFile(OpenBlockFile(pos, true)).IsNull()); + BOOST_CHECK(!AutoFile(blockman.OpenBlockFile(pos, true)).IsNull()); // Check that the file is unlinked after ScanAndUnlinkAlreadyPrunedFiles // once m_have_pruned is set blockman.m_have_pruned = true; WITH_LOCK(chainman->GetMutex(), blockman.ScanAndUnlinkAlreadyPrunedFiles()); - BOOST_CHECK(AutoFile(OpenBlockFile(pos, true)).IsNull()); + BOOST_CHECK(AutoFile(blockman.OpenBlockFile(pos, true)).IsNull()); // Check that calling with already pruned files doesn't cause an error WITH_LOCK(chainman->GetMutex(), blockman.ScanAndUnlinkAlreadyPrunedFiles()); @@ -79,7 +83,7 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_scan_unlink_already_pruned_files, TestChain BOOST_CHECK_NE(old_tip, new_tip); const int new_file_number{WITH_LOCK(chainman->GetMutex(), return new_tip->GetBlockPos().nFile)}; const FlatFilePos new_pos(new_file_number, 0); - BOOST_CHECK(!AutoFile(OpenBlockFile(new_pos, true)).IsNull()); + BOOST_CHECK(!AutoFile(blockman.OpenBlockFile(new_pos, true)).IsNull()); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 5d4c5eea0e..93c0412593 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -5,6 +5,7 @@ #include <common/bloom.h> #include <clientversion.h> +#include <common/system.h> #include <key.h> #include <key_io.h> #include <merkleblock.h> @@ -16,7 +17,6 @@ #include <test/util/setup_common.h> #include <uint256.h> #include <util/strencodings.h> -#include <util/system.h> #include <vector> diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 9011e703e8..cb3831071a 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -7,6 +7,7 @@ #include <sync.h> #include <test/util/random.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <util/time.h> #include <boost/test/unit_test.hpp> @@ -27,9 +28,9 @@ struct NoLockLoggingTestingSetup : public TestingSetup { NoLockLoggingTestingSetup() #ifdef DEBUG_LOCKCONTENTION - : TestingSetup{CBaseChainParams::MAIN, /*extra_args=*/{"-debugexclude=lock"}} {} + : TestingSetup{ChainType::MAIN, /*extra_args=*/{"-debugexclude=lock"}} {} #else - : TestingSetup{CBaseChainParams::MAIN} {} + : TestingSetup{ChainType::MAIN} {} #endif }; diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index c4b2b4c63b..7a7790e2ae 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -127,7 +127,7 @@ std::set<std::pair<CPubKey, KeyOriginInfo>> GetKeyOriginData(const FlatSigningPr return ret; } -void DoCheck(const std::string& prv, const std::string& pub, const std::string& norm_pub, int flags, +void DoCheck(std::string prv, std::string pub, const std::string& norm_pub, int flags, const std::vector<std::vector<std::string>>& scripts, const std::optional<OutputType>& type, const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY, bool replace_apostrophe_with_h_in_prv=false, bool replace_apostrophe_with_h_in_pub=false, uint32_t spender_nlocktime=0, uint32_t spender_nsequence=CTxIn::SEQUENCE_FINAL, @@ -141,16 +141,14 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& std::unique_ptr<Descriptor> parse_pub; // Check that parsing succeeds. if (replace_apostrophe_with_h_in_prv) { - parse_priv = Parse(UseHInsteadOfApostrophe(prv), keys_priv, error); - } else { - parse_priv = Parse(prv, keys_priv, error); + prv = UseHInsteadOfApostrophe(prv); } + parse_priv = Parse(prv, keys_priv, error); BOOST_CHECK_MESSAGE(parse_priv, error); if (replace_apostrophe_with_h_in_pub) { - parse_pub = Parse(UseHInsteadOfApostrophe(pub), keys_pub, error); - } else { - parse_pub = Parse(pub, keys_pub, error); + pub = UseHInsteadOfApostrophe(pub); } + parse_pub = Parse(pub, keys_pub, error); BOOST_CHECK_MESSAGE(parse_pub, error); // Check that the correct OutputType is inferred @@ -171,19 +169,19 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& if (!(flags & MISSING_PRIVKEYS)) { std::string prv1; BOOST_CHECK(parse_priv->ToPrivateString(keys_priv, prv1)); - BOOST_CHECK(EqualDescriptor(prv, prv1)); + BOOST_CHECK_MESSAGE(EqualDescriptor(prv, prv1), "Private ser: " + prv1 + " Private desc: " + prv); BOOST_CHECK(!parse_priv->ToPrivateString(keys_pub, prv1)); BOOST_CHECK(parse_pub->ToPrivateString(keys_priv, prv1)); - BOOST_CHECK(EqualDescriptor(prv, prv1)); + BOOST_CHECK_MESSAGE(EqualDescriptor(prv, prv1), "Private ser: " + prv1 + " Private desc: " + prv); BOOST_CHECK(!parse_pub->ToPrivateString(keys_pub, prv1)); } // Check that private can produce the normalized descriptors std::string norm1; BOOST_CHECK(parse_priv->ToNormalizedString(keys_priv, norm1)); - BOOST_CHECK(EqualDescriptor(norm1, norm_pub)); + BOOST_CHECK_MESSAGE(EqualDescriptor(norm1, norm_pub), "priv->ToNormalizedString(): " + norm1 + " Norm. desc: " + norm_pub); BOOST_CHECK(parse_pub->ToNormalizedString(keys_priv, norm1)); - BOOST_CHECK(EqualDescriptor(norm1, norm_pub)); + BOOST_CHECK_MESSAGE(EqualDescriptor(norm1, norm_pub), "pub->ToNormalizedString(): " + norm1 + " Norm. desc: " + norm_pub); // Check whether IsRange on both returns the expected result BOOST_CHECK_EQUAL(parse_pub->IsRange(), (flags & RANGE) != 0); @@ -357,32 +355,13 @@ void Check(const std::string& prv, const std::string& pub, const std::string& no const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY, uint32_t spender_nlocktime=0, uint32_t spender_nsequence=CTxIn::SEQUENCE_FINAL, std::map<std::vector<uint8_t>, std::vector<uint8_t>> preimages={}) { - bool found_apostrophes_in_prv = false; - bool found_apostrophes_in_pub = false; - // Do not replace apostrophes with 'h' in prv and pub DoCheck(prv, pub, norm_pub, flags, scripts, type, paths, /*replace_apostrophe_with_h_in_prv=*/false, /*replace_apostrophe_with_h_in_pub=*/false, /*spender_nlocktime=*/spender_nlocktime, /*spender_nsequence=*/spender_nsequence, /*preimages=*/preimages); - // Replace apostrophes with 'h' in prv but not in pub, if apostrophes are found in prv - if (prv.find('\'') != std::string::npos) { - found_apostrophes_in_prv = true; - DoCheck(prv, pub, norm_pub, flags, scripts, type, paths, /*replace_apostrophe_with_h_in_prv=*/true, - /*replace_apostrophe_with_h_in_pub=*/false, /*spender_nlocktime=*/spender_nlocktime, - /*spender_nsequence=*/spender_nsequence, /*preimages=*/preimages); - } - - // Replace apostrophes with 'h' in pub but not in prv, if apostrophes are found in pub - if (pub.find('\'') != std::string::npos) { - found_apostrophes_in_pub = true; - DoCheck(prv, pub, norm_pub, flags, scripts, type, paths, /*replace_apostrophe_with_h_in_prv=*/false, - /*replace_apostrophe_with_h_in_pub=*/true, /*spender_nlocktime=*/spender_nlocktime, - /*spender_nsequence=*/spender_nsequence, /*preimages=*/preimages); - } - // Replace apostrophes with 'h' both in prv and in pub, if apostrophes are found in both - if (found_apostrophes_in_prv && found_apostrophes_in_pub) { + if (prv.find('\'') != std::string::npos && pub.find('\'') != std::string::npos) { DoCheck(prv, pub, norm_pub, flags, scripts, type, paths, /*replace_apostrophe_with_h_in_prv=*/true, /*replace_apostrophe_with_h_in_pub=*/true, /*spender_nlocktime=*/spender_nlocktime, /*spender_nsequence=*/spender_nsequence, /*preimages=*/preimages); @@ -398,12 +377,12 @@ BOOST_AUTO_TEST_CASE(descriptor_test) // Basic single-key compressed Check("combo(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "combo(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "combo(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bdac","76a9149a1c78a507689f6f54b847ad1cef1e614ee23f1e88ac","00149a1c78a507689f6f54b847ad1cef1e614ee23f1e","a91484ab21b1b2fd065d4504ff693d832434b6108d7b87"}}, std::nullopt); Check("pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bdac"}}, std::nullopt); - Check("pkh([deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh([deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"76a9149a1c78a507689f6f54b847ad1cef1e614ee23f1e88ac"}}, OutputType::LEGACY, {{1,0x80000002UL,3,0x80000004UL}}); + Check("pkh([deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh([deadbeef/1/2h/3/4h]03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"76a9149a1c78a507689f6f54b847ad1cef1e614ee23f1e88ac"}}, OutputType::LEGACY, {{1,0x80000002UL,3,0x80000004UL}}); Check("wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"00149a1c78a507689f6f54b847ad1cef1e614ee23f1e"}}, OutputType::BECH32); Check("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a91484ab21b1b2fd065d4504ff693d832434b6108d7b87"}}, OutputType::P2SH_SEGWIT); Check("tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE | XONLY_KEYS, {{"512077aab6e066f8a7419c5ab714c12c67d25007ed55a43cadcacb4d7a970a093f11"}}, OutputType::BECH32M); CheckUnparsable("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY2))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5))", "wpkh(): Pubkey '03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5' is invalid"); // Invalid pubkey - CheckUnparsable("pkh(deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh(deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh(): Key origin start '[ character expected but not found, got 'd' instead"); // Missing start bracket in key origin + CheckUnparsable("pkh(deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh(deadbeef/1/2h/3/4h]03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh(): Key origin start '[ character expected but not found, got 'd' instead"); // Missing start bracket in key origin CheckUnparsable("pkh([deadbeef]/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef]/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh(): Multiple ']' characters found for a single pubkey"); // Multiple end brackets in key origin // Basic single-key uncompressed @@ -426,10 +405,10 @@ BOOST_AUTO_TEST_CASE(descriptor_test) // Versions with BIP32 derivations Check("combo([01234567]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([01234567]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "combo([01234567]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", SIGNABLE, {{"2102d2b36900396c9282fa14628566582f206a5dd0bcc8d5e892611806cafb0301f0ac","76a91431a507b815593dfc51ffc7245ae7e5aee304246e88ac","001431a507b815593dfc51ffc7245ae7e5aee304246e","a9142aafb926eb247cb18240a7f4c07983ad1f37922687"}}, std::nullopt); Check("pk(xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)", "pk(xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)", "pk(xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)", DEFAULT, {{"210379e45b3cf75f9c5f9befd8e9506fb962f6a9d185ac87001ec44a8d3df8d4a9e3ac"}}, std::nullopt, {{0}}); - Check("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([bd16bee5/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{0xFFFFFFFFUL,0}}); + Check("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([bd16bee5/2147483647h]xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{0xFFFFFFFFUL,0}}); - Check("wpkh([ffffffff/13']xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", RANGE, {{"0014326b2249e3a25d5dc60935f044ee835d090ba859"},{"0014af0bd98abc2f2cae66e36896a39ffe2d32984fb7"},{"00141fa798efd1cbf95cebf912c031b8a4a6e9fb9f27"}}, OutputType::BECH32, {{0x8000000DUL, 1, 2, 0}, {0x8000000DUL, 1, 2, 1}, {0x8000000DUL, 1, 2, 2}}); - Check("sh(wpkh(xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", RANGE | HARDENED | DERIVE_HARDENED, {{"a9149a4d9901d6af519b2a23d4a2f51650fcba87ce7b87"},{"a914bed59fc0024fae941d6e20a3b44a109ae740129287"},{"a9148483aa1116eb9c05c482a72bada4b1db24af654387"}}, OutputType::P2SH_SEGWIT, {{10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); + Check("wpkh([ffffffff/13']xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", "wpkh([ffffffff/13h]xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", RANGE, {{"0014326b2249e3a25d5dc60935f044ee835d090ba859"},{"0014af0bd98abc2f2cae66e36896a39ffe2d32984fb7"},{"00141fa798efd1cbf95cebf912c031b8a4a6e9fb9f27"}}, OutputType::BECH32, {{0x8000000DUL, 1, 2, 0}, {0x8000000DUL, 1, 2, 1}, {0x8000000DUL, 1, 2, 2}}); + Check("sh(wpkh(xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", RANGE | HARDENED | DERIVE_HARDENED, {{"a9149a4d9901d6af519b2a23d4a2f51650fcba87ce7b87"},{"a914bed59fc0024fae941d6e20a3b44a109ae740129287"},{"a9148483aa1116eb9c05c482a72bada4b1db24af654387"}}, OutputType::P2SH_SEGWIT, {{10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); Check("combo(xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*)", "combo(xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*)", "combo(xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*)", RANGE, {{"2102df12b7035bdac8e3bab862a3a83d06ea6b17b6753d52edecba9be46f5d09e076ac","76a914f90e3178ca25f2c808dc76624032d352fdbdfaf288ac","0014f90e3178ca25f2c808dc76624032d352fdbdfaf2","a91408f3ea8c68d4a7585bf9e8bda226723f70e445f087"},{"21032869a233c9adff9a994e4966e5b821fd5bac066da6c3112488dc52383b4a98ecac","76a914a8409d1b6dfb1ed2a3e8aa5e0ef2ff26b15b75b788ac","0014a8409d1b6dfb1ed2a3e8aa5e0ef2ff26b15b75b7","a91473e39884cb71ae4e5ac9739e9225026c99763e6687"}}, std::nullopt, {{0}, {1}}); Check("tr(xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/0/*,pk(xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/1/*))", "tr(xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/0/*,pk(xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/*))", "tr(xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/0/*,pk(xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/*))", XONLY_KEYS | RANGE, {{"512078bc707124daa551b65af74de2ec128b7525e10f374dc67b64e00ce0ab8b3e12"}, {"512001f0a02a17808c20134b78faab80ef93ffba82261ccef0a2314f5d62b6438f11"}, {"512021024954fcec88237a9386fce80ef2ced5f1e91b422b26c59ccfc174c8d1ad25"}}, OutputType::BECH32M, {{0, 0}, {0, 1}, {0, 2}, {1, 0}, {1, 1}, {1, 2}}); // Mixed xpubs and const pubkeys @@ -440,23 +419,23 @@ BOOST_AUTO_TEST_CASE(descriptor_test) CheckUnparsable("combo([012345678]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([012345678]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "combo(): Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long key fingerprint CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483648)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483648)", "pkh(): Key path value 2147483648 is out of range"); // BIP 32 path element overflow CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/1aa)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1aa)", "pkh(): Key path value '1aa' is not a valid uint32"); // Path is not valid uint - Check("pkh([01234567/10/20]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh([01234567/10/20]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([01234567/10/20/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{10, 20, 0xFFFFFFFFUL, 0}}); + Check("pkh([01234567/10/20]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh([01234567/10/20]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([01234567/10/20/2147483647h]xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{10, 20, 0xFFFFFFFFUL, 0}}); // Multisig constructions Check("multi(1,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "multi(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, std::nullopt); Check("sortedmulti(1,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "sortedmulti(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "sortedmulti(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, std::nullopt); Check("sortedmulti(1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "sortedmulti(1,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "sortedmulti(1,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, std::nullopt); - Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", "sh(multi(2,[00000000/111h/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); Check("sortedmulti(2,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/*,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0/0/*)", "sortedmulti(2,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/*,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0/0/*)", "sortedmulti(2,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/*,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0/0/*)", RANGE, {{"5221025d5fc65ebb8d44a5274b53bac21ff8307fec2334a32df05553459f8b1f7fe1b62102fbd47cc8034098f0e6a94c6aeee8528abf0a2153a5d8e46d325b7284c046784652ae"}, {"52210264fd4d1f5dea8ded94c61e9641309349b62f27fbffe807291f664e286bfbe6472103f4ece6dfccfa37b211eb3d0af4d0c61dba9ef698622dc17eecdf764beeb005a652ae"}, {"5221022ccabda84c30bad578b13c89eb3b9544ce149787e5b538175b1d1ba259cbb83321024d902e1a2fc7a8755ab5b694c575fce742c48d9ff192e63df5193e4c7afe1f9c52ae"}}, std::nullopt, {{0}, {1}, {2}, {0, 0, 0}, {0, 0, 1}, {0, 0, 2}}); - Check("wsh(multi(2,xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "wsh(multi(2,[bd16bee5/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", HARDENED | RANGE | DERIVE_HARDENED, {{"0020b92623201f3bb7c3771d45b2ad1d0351ea8fbf8cfe0a0e570264e1075fa1948f"},{"002036a08bbe4923af41cf4316817c93b8d37e2f635dd25cfff06bd50df6ae7ea203"},{"0020a96e7ab4607ca6b261bfe3245ffda9c746b28d3f59e83d34820ec0e2b36c139c"}}, OutputType::BECH32, {{0xFFFFFFFFUL,0}, {1,2,0}, {1,2,1}, {1,2,2}, {10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); + Check("wsh(multi(2,xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "wsh(multi(2,[bd16bee5/2147483647h]xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", HARDENED | RANGE | DERIVE_HARDENED, {{"0020b92623201f3bb7c3771d45b2ad1d0351ea8fbf8cfe0a0e570264e1075fa1948f"},{"002036a08bbe4923af41cf4316817c93b8d37e2f635dd25cfff06bd50df6ae7ea203"},{"0020a96e7ab4607ca6b261bfe3245ffda9c746b28d3f59e83d34820ec0e2b36c139c"}}, OutputType::BECH32, {{0xFFFFFFFFUL,0}, {1,2,0}, {1,2,1}, {1,2,2}, {10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); Check("sh(wsh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9)))","sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", "sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", SIGNABLE, {{"a9147fc63e13dc25e8a95a3cee3d9a714ac3afd96f1e87"}}, OutputType::P2SH_SEGWIT); Check("tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,pk(KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy))", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,pk(669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0))", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,pk(669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0))", SIGNABLE | XONLY_KEYS, {{"512017cf18db381d836d8923b1bdb246cfcd818da1a9f0e6e7907f187f0b2f937754"}}, OutputType::BECH32M); CheckUnparsable("sh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9))","sh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232))", "P2SH script is too large, 547 bytes is larger than 520 bytes"); // P2SH does not fit 16 compressed pubkeys in a redeemscript - CheckUnparsable("wsh(multi(2,[aaaaaaaa][aaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa][aaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Multiple ']' characters found for a single pubkey"); // Double key origin descriptor - CheckUnparsable("wsh(multi(2,[aaaagaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaagaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint 'aaagaaaa' is not hex"); // Non hex fingerprint - CheckUnparsable("wsh(multi(2,[aaaaaaaa],xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa],xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: No key provided"); // No public key with origin - CheckUnparsable("wsh(multi(2,[aaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint is not 4 bytes (7 characters instead of 8 characters)"); // Too short fingerprint - CheckUnparsable("wsh(multi(2,[aaaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaa][aaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa][aaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647h/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", "Multi: Multiple ']' characters found for a single pubkey"); // Double key origin descriptor + CheckUnparsable("wsh(multi(2,[aaaagaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaagaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647h/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", "Multi: Fingerprint 'aaagaaaa' is not hex"); // Non hex fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaa],xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa],xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", "Multi: No key provided"); // No public key with origin + CheckUnparsable("wsh(multi(2,[aaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647h/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", "Multi: Fingerprint is not 4 bytes (7 characters instead of 8 characters)"); // Too short fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647h/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", "Multi: Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long fingerprint CheckUnparsable("multi(a,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(a,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multi threshold 'a' is not valid"); // Invalid threshold CheckUnparsable("multi(0,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(0,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multisig threshold cannot be 0, must be at least 1"); // Threshold of 0 CheckUnparsable("multi(3,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(3,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multisig threshold cannot be larger than the number of keys; threshold is 3 but only 2 keys specified"); // Threshold larger than number of keys @@ -474,8 +453,8 @@ BOOST_AUTO_TEST_CASE(descriptor_test) CheckUnparsable("wsh(wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "wsh(wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))", "Can only have wsh() at top level or inside sh()"); // Cannot embed P2WSH inside P2WSH // Checksums - Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); - Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", "sh(multi(2,[00000000/111h/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#hgmsckna", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", "sh(multi(2,[00000000/111h/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#", "Expected 8 character checksum, not 0 characters"); // Empty checksum CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfyq", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5tq", "Expected 8 character checksum, not 9 characters"); // Too long checksum CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxf", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5", "Expected 8 character checksum, not 7 characters"); // Too short checksum @@ -491,13 +470,12 @@ BOOST_AUTO_TEST_CASE(descriptor_test) Check( "rawtr(xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/86'/1'/0'/1/*)#a5gn3t7k", "rawtr(xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/86'/1'/0'/1/*)#4ur3xhft", - "rawtr([5a61ff8e/86'/1'/0']xpub6DtZpc9PRL2B6pwoNGysmHAaBofDmWv5S6KQEKKGPKhf5fV62ywDtSziSApYVK3JnYY5KUSgiCwiXW5wtd8z7LNBxT9Mu5sEro8itdGfTeA/1/*)#llheyd9x", + "rawtr([5a61ff8e/86h/1h/0h]xpub6DtZpc9PRL2B6pwoNGysmHAaBofDmWv5S6KQEKKGPKhf5fV62ywDtSziSApYVK3JnYY5KUSgiCwiXW5wtd8z7LNBxT9Mu5sEro8itdGfTeA/1/*)#vwgx7hj9", RANGE | HARDENED | XONLY_KEYS, {{"51205172af752f057d543ce8e4a6f8dcf15548ec6be44041bfa93b72e191cfc8c1ee"}, {"51201b66f20b86f700c945ecb9ad9b0ad1662b73084e2bfea48bee02126350b8a5b1"}, {"512063e70f66d815218abcc2306aa930aaca07c5cde73b75127eb27b5e8c16b58a25"}}, OutputType::BECH32M, {{0x80000056, 0x80000001, 0x80000000, 1, 0}, {0x80000056, 0x80000001, 0x80000000, 1, 1}, {0x80000056, 0x80000001, 0x80000000, 1, 2}}); - Check( "rawtr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "rawtr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index 5ad7a25c53..7218bb82d1 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -16,6 +16,7 @@ #include <test/util/setup_common.h> #include <time.h> #include <util/asmap.h> +#include <util/chaintype.h> #include <cassert> #include <cstdint> @@ -34,7 +35,7 @@ int32_t GetCheckRatio() void initialize_addrman() { - static const auto testing_setup = MakeNoLogFileContext<>(CBaseChainParams::REGTEST); + static const auto testing_setup = MakeNoLogFileContext<>(ChainType::REGTEST); g_setup = testing_setup.get(); } diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index c3e17724eb..e90dcc189a 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -11,6 +11,7 @@ #include <pubkey.h> #include <streams.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> #include <validation.h> #include <version.h> @@ -19,7 +20,7 @@ void initialize_block() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(block, initialize_block) diff --git a/src/test/fuzz/buffered_file.cpp b/src/test/fuzz/buffered_file.cpp index 67cac8fa4e..2f7ce60c7f 100644 --- a/src/test/fuzz/buffered_file.cpp +++ b/src/test/fuzz/buffered_file.cpp @@ -53,7 +53,7 @@ FUZZ_TARGET(buffered_file) return; } try { - opt_buffered_file->FindByte(fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + opt_buffered_file->FindByte(std::byte(fuzzed_data_provider.ConsumeIntegral<uint8_t>())); } catch (const std::ios_base::failure&) { } }, diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 5843b80c0a..fc7e000dc7 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <chainparamsbase.h> #include <coins.h> #include <consensus/amount.h> #include <consensus/tx_check.h> diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 0f3c850e66..f81658b832 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -4,7 +4,6 @@ #include <addrman.h> #include <chainparams.h> -#include <chainparamsbase.h> #include <common/args.h> #include <net.h> #include <netaddress.h> diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index 1f5601ca9f..12c22ef2ed 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -6,11 +6,12 @@ #include <pubkey.h> #include <script/descriptor.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> void initialize_descriptor_parse() { ECC_Start(); - SelectParams(CBaseChainParams::MAIN); + SelectParams(ChainType::MAIN); } FUZZ_TARGET_INIT(descriptor_parse, initialize_descriptor_parse) diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 0bbfb206d5..44ba8bc254 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -14,14 +14,19 @@ #include <csignal> #include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> #include <exception> #include <fstream> #include <functional> +#include <iostream> #include <map> #include <memory> #include <string> #include <tuple> #include <unistd.h> +#include <utility> #include <vector> const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; @@ -77,13 +82,13 @@ void initialize() return WrappedGetAddrInfo(name, false); }; - bool should_abort{false}; + bool should_exit{false}; if (std::getenv("PRINT_ALL_FUZZ_TARGETS_AND_ABORT")) { for (const auto& t : FuzzTargets()) { if (std::get<2>(t.second)) continue; std::cout << t.first << std::endl; } - should_abort = true; + should_exit = true; } if (const char* out_path = std::getenv("WRITE_ALL_FUZZ_TARGETS_AND_ABORT")) { std::cout << "Writing all fuzz target names to '" << out_path << "'." << std::endl; @@ -92,13 +97,23 @@ void initialize() if (std::get<2>(t.second)) continue; out_stream << t.first << std::endl; } - should_abort = true; + should_exit= true; + } + if (should_exit){ + std::exit(EXIT_SUCCESS); + } + if (const auto* env_fuzz{std::getenv("FUZZ")}) { + // To allow for easier fuzz executable binary modification, + static std::string g_copy{env_fuzz}; // create copy to avoid compiler optimizations, and + g_fuzz_target = g_copy.c_str(); // strip string after the first null-char. + } else { + std::cerr << "Must select fuzz target with the FUZZ env var." << std::endl; + std::cerr << "Hint: Set the PRINT_ALL_FUZZ_TARGETS_AND_ABORT=1 env var to see all compiled targets." << std::endl; + std::exit(EXIT_FAILURE); } - Assert(!should_abort); - g_fuzz_target = Assert(std::getenv("FUZZ")); const auto it = FuzzTargets().find(g_fuzz_target); if (it == FuzzTargets().end()) { - std::cerr << "No fuzzer for " << g_fuzz_target << "." << std::endl; + std::cerr << "No fuzz target compiled for " << g_fuzz_target << "." << std::endl; std::exit(EXIT_FAILURE); } Assert(!g_test_one_input); diff --git a/src/test/fuzz/headerssync.cpp b/src/test/fuzz/headerssync.cpp index 521afadb51..c1a187038b 100644 --- a/src/test/fuzz/headerssync.cpp +++ b/src/test/fuzz/headerssync.cpp @@ -6,6 +6,7 @@ #include <test/fuzz/util.h> #include <test/util/setup_common.h> #include <uint256.h> +#include <util/chaintype.h> #include <util/time.h> #include <validation.h> @@ -15,7 +16,7 @@ static void initialize_headers_sync_state_fuzz() { static const auto testing_setup = MakeNoLogFileContext<>( - /*chain_name=*/CBaseChainParams::MAIN); + /*chain_type=*/ChainType::MAIN); } void MakeHeadersContinuous( diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index ead877fe05..edb1dca457 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -4,6 +4,7 @@ #include <arith_uint256.h> #include <common/args.h> +#include <common/system.h> #include <compressor.h> #include <consensus/amount.h> #include <consensus/merkle.h> @@ -26,12 +27,12 @@ #include <test/fuzz/util.h> #include <uint256.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/check.h> #include <util/moneystr.h> #include <util/overflow.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <version.h> #include <cassert> @@ -42,7 +43,7 @@ void initialize_integer() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(integer, initialize_integer) diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index ea6883c08d..3eab2e20c0 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <chainparamsbase.h> #include <key.h> #include <key_io.h> #include <outputtype.h> @@ -17,6 +16,7 @@ #include <script/standard.h> #include <streams.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <cassert> @@ -28,7 +28,7 @@ void initialize_key() { ECC_Start(); - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(key, initialize_key) diff --git a/src/test/fuzz/key_io.cpp b/src/test/fuzz/key_io.cpp index 29c6996365..a1c587a75b 100644 --- a/src/test/fuzz/key_io.cpp +++ b/src/test/fuzz/key_io.cpp @@ -5,6 +5,7 @@ #include <chainparams.h> #include <key_io.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> #include <cassert> #include <cstdint> @@ -14,7 +15,7 @@ void initialize_key_io() { ECC_Start(); - SelectParams(CBaseChainParams::MAIN); + SelectParams(ChainType::MAIN); } FUZZ_TARGET_INIT(key_io, initialize_key_io) diff --git a/src/test/fuzz/message.cpp b/src/test/fuzz/message.cpp index 63e24aacdd..8b7e3f11cc 100644 --- a/src/test/fuzz/message.cpp +++ b/src/test/fuzz/message.cpp @@ -7,6 +7,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/chaintype.h> #include <util/message.h> #include <util/strencodings.h> @@ -19,7 +20,7 @@ void initialize_message() { ECC_Start(); - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(message, initialize_message) diff --git a/src/test/fuzz/mini_miner.cpp b/src/test/fuzz/mini_miner.cpp new file mode 100644 index 0000000000..2b371f6d5f --- /dev/null +++ b/src/test/fuzz/mini_miner.cpp @@ -0,0 +1,193 @@ +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/fuzz/util/mempool.h> +#include <test/util/script.h> +#include <test/util/setup_common.h> +#include <test/util/txmempool.h> +#include <test/util/mining.h> + +#include <node/mini_miner.h> +#include <node/miner.h> +#include <primitives/transaction.h> +#include <random.h> +#include <txmempool.h> + +#include <deque> +#include <vector> + +namespace { + +const TestingSetup* g_setup; +std::deque<COutPoint> g_available_coins; +void initialize_miner() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); + for (uint32_t i = 0; i < uint32_t{100}; ++i) { + g_available_coins.push_back(COutPoint{uint256::ZERO, i}); + } +} + +// Test that the MiniMiner can run with various outpoints and feerates. +FUZZ_TARGET_INIT(mini_miner, initialize_miner) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CTxMemPool pool{CTxMemPool::Options{}}; + std::vector<COutPoint> outpoints; + std::deque<COutPoint> available_coins = g_available_coins; + LOCK2(::cs_main, pool.cs); + // Cluster size cannot exceed 500 + LIMITED_WHILE(!available_coins.empty(), 500) + { + CMutableTransaction mtx = CMutableTransaction(); + const size_t num_inputs = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, available_coins.size()); + const size_t num_outputs = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 50); + for (size_t n{0}; n < num_inputs; ++n) { + auto prevout = available_coins.front(); + mtx.vin.push_back(CTxIn(prevout, CScript())); + available_coins.pop_front(); + } + for (uint32_t n{0}; n < num_outputs; ++n) { + mtx.vout.push_back(CTxOut(100, P2WSH_OP_TRUE)); + } + CTransactionRef tx = MakeTransactionRef(mtx); + TestMemPoolEntryHelper entry; + const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; + assert(MoneyRange(fee)); + pool.addUnchecked(entry.Fee(fee).FromTx(tx)); + + // All outputs are available to spend + for (uint32_t n{0}; n < num_outputs; ++n) { + if (fuzzed_data_provider.ConsumeBool()) { + available_coins.push_back(COutPoint{tx->GetHash(), n}); + } + } + + if (fuzzed_data_provider.ConsumeBool() && !tx->vout.empty()) { + // Add outpoint from this tx (may or not be spent by a later tx) + outpoints.push_back(COutPoint{tx->GetHash(), + (uint32_t)fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, tx->vout.size())}); + } else { + // Add some random outpoint (will be interpreted as confirmed or not yet submitted + // to mempool). + auto outpoint = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (outpoint.has_value() && std::find(outpoints.begin(), outpoints.end(), *outpoint) == outpoints.end()) { + outpoints.push_back(*outpoint); + } + } + + } + + const CFeeRate target_feerate{CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/1000)}}; + std::optional<CAmount> total_bumpfee; + CAmount sum_fees = 0; + { + node::MiniMiner mini_miner{pool, outpoints}; + assert(mini_miner.IsReadyToCalculate()); + const auto bump_fees = mini_miner.CalculateBumpFees(target_feerate); + for (const auto& outpoint : outpoints) { + auto it = bump_fees.find(outpoint); + assert(it != bump_fees.end()); + assert(it->second >= 0); + sum_fees += it->second; + } + assert(!mini_miner.IsReadyToCalculate()); + } + { + node::MiniMiner mini_miner{pool, outpoints}; + assert(mini_miner.IsReadyToCalculate()); + total_bumpfee = mini_miner.CalculateTotalBumpFees(target_feerate); + assert(total_bumpfee.has_value()); + assert(!mini_miner.IsReadyToCalculate()); + } + // Overlapping ancestry across multiple outpoints can only reduce the total bump fee. + assert (sum_fees >= *total_bumpfee); +} + +// Test that MiniMiner and BlockAssembler build the same block given the same transactions and constraints. +FUZZ_TARGET_INIT(mini_miner_selection, initialize_miner) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CTxMemPool pool{CTxMemPool::Options{}}; + // Make a copy to preserve determinism. + std::deque<COutPoint> available_coins = g_available_coins; + std::vector<CTransactionRef> transactions; + + LOCK2(::cs_main, pool.cs); + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) + { + CMutableTransaction mtx = CMutableTransaction(); + assert(!available_coins.empty()); + const size_t num_inputs = std::min(size_t{2}, available_coins.size()); + const size_t num_outputs = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(2, 5); + for (size_t n{0}; n < num_inputs; ++n) { + auto prevout = available_coins.at(0); + mtx.vin.push_back(CTxIn(prevout, CScript())); + available_coins.pop_front(); + } + for (uint32_t n{0}; n < num_outputs; ++n) { + mtx.vout.push_back(CTxOut(100, P2WSH_OP_TRUE)); + } + CTransactionRef tx = MakeTransactionRef(mtx); + + // First 2 outputs are available to spend. The rest are added to outpoints to calculate bumpfees. + // There is no overlap between spendable coins and outpoints passed to MiniMiner because the + // MiniMiner interprets spent coins as to-be-replaced and excludes them. + for (uint32_t n{0}; n < num_outputs - 1; ++n) { + if (fuzzed_data_provider.ConsumeBool()) { + available_coins.push_front(COutPoint{tx->GetHash(), n}); + } else { + available_coins.push_back(COutPoint{tx->GetHash(), n}); + } + } + + // Stop if pool reaches DEFAULT_BLOCK_MAX_WEIGHT because BlockAssembler will stop when the + // block template reaches that, but the MiniMiner will keep going. + if (pool.GetTotalTxSize() + GetVirtualTransactionSize(*tx) >= DEFAULT_BLOCK_MAX_WEIGHT) break; + TestMemPoolEntryHelper entry; + const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; + assert(MoneyRange(fee)); + pool.addUnchecked(entry.Fee(fee).FromTx(tx)); + transactions.push_back(tx); + } + std::vector<COutPoint> outpoints; + for (const auto& coin : g_available_coins) { + if (!pool.GetConflictTx(coin)) outpoints.push_back(coin); + } + for (const auto& tx : transactions) { + assert(pool.exists(GenTxid::Txid(tx->GetHash()))); + for (uint32_t n{0}; n < tx->vout.size(); ++n) { + COutPoint coin{tx->GetHash(), n}; + if (!pool.GetConflictTx(coin)) outpoints.push_back(coin); + } + } + const CFeeRate target_feerate{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; + + node::BlockAssembler::Options miner_options; + miner_options.blockMinFeeRate = target_feerate; + miner_options.nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; + miner_options.test_block_validity = false; + + node::BlockAssembler miner{g_setup->m_node.chainman->ActiveChainstate(), &pool, miner_options}; + node::MiniMiner mini_miner{pool, outpoints}; + assert(mini_miner.IsReadyToCalculate()); + + CScript spk_placeholder = CScript() << OP_0; + // Use BlockAssembler as oracle. BlockAssembler and MiniMiner should select the same + // transactions, stopping once packages do not meet target_feerate. + const auto blocktemplate{miner.CreateNewBlock(spk_placeholder)}; + mini_miner.BuildMockTemplate(target_feerate); + assert(!mini_miner.IsReadyToCalculate()); + auto mock_template_txids = mini_miner.GetMockTemplateTxids(); + // MiniMiner doesn't add a coinbase tx. + assert(mock_template_txids.count(blocktemplate->block.vtx[0]->GetHash()) == 0); + mock_template_txids.emplace(blocktemplate->block.vtx[0]->GetHash()); + assert(mock_template_txids.size() <= blocktemplate->block.vtx.size()); + assert(mock_template_txids.size() >= blocktemplate->block.vtx.size()); + assert(mock_template_txids.size() == blocktemplate->block.vtx.size()); + for (const auto& tx : blocktemplate->block.vtx) { + assert(mock_template_txids.count(tx->GetHash())); + } +} +} // namespace diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index 13b4638688..e090f13061 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <chainparamsbase.h> #include <net.h> #include <net_permissions.h> #include <netaddress.h> @@ -16,6 +15,7 @@ #include <test/util/net.h> #include <test/util/setup_common.h> #include <util/asmap.h> +#include <util/chaintype.h> #include <cstdint> #include <optional> @@ -24,7 +24,7 @@ void initialize_net() { - static const auto testing_setup = MakeNoLogFileContext<>(CBaseChainParams::MAIN); + static const auto testing_setup = MakeNoLogFileContext<>(ChainType::MAIN); } FUZZ_TARGET_INIT(net, initialize_net) diff --git a/src/test/fuzz/netbase_dns_lookup.cpp b/src/test/fuzz/netbase_dns_lookup.cpp index 81e216b358..dcf500acc3 100644 --- a/src/test/fuzz/netbase_dns_lookup.cpp +++ b/src/test/fuzz/netbase_dns_lookup.cpp @@ -29,33 +29,29 @@ FUZZ_TARGET(netbase_dns_lookup) }; { - std::vector<CNetAddr> resolved_addresses; - if (LookupHost(name, resolved_addresses, max_results, allow_lookup, fuzzed_dns_lookup_function)) { - for (const CNetAddr& resolved_address : resolved_addresses) { - assert(!resolved_address.IsInternal()); - } + const std::vector<CNetAddr> resolved_addresses{LookupHost(name, max_results, allow_lookup, fuzzed_dns_lookup_function)}; + for (const CNetAddr& resolved_address : resolved_addresses) { + assert(!resolved_address.IsInternal()); } assert(resolved_addresses.size() <= max_results || max_results == 0); } { - CNetAddr resolved_address; - if (LookupHost(name, resolved_address, allow_lookup, fuzzed_dns_lookup_function)) { - assert(!resolved_address.IsInternal()); + const std::optional<CNetAddr> resolved_address{LookupHost(name, allow_lookup, fuzzed_dns_lookup_function)}; + if (resolved_address.has_value()) { + assert(!resolved_address.value().IsInternal()); } } { - std::vector<CService> resolved_services; - if (Lookup(name, resolved_services, default_port, allow_lookup, max_results, fuzzed_dns_lookup_function)) { - for (const CNetAddr& resolved_service : resolved_services) { - assert(!resolved_service.IsInternal()); - } + const std::vector<CService> resolved_services{Lookup(name, default_port, allow_lookup, max_results, fuzzed_dns_lookup_function)}; + for (const CNetAddr& resolved_service : resolved_services) { + assert(!resolved_service.IsInternal()); } assert(resolved_services.size() <= max_results || max_results == 0); } { - CService resolved_service; - if (Lookup(name, resolved_service, default_port, allow_lookup, fuzzed_dns_lookup_function)) { - assert(!resolved_service.IsInternal()); + const std::optional<CService> resolved_service{Lookup(name, default_port, allow_lookup, fuzzed_dns_lookup_function)}; + if (resolved_service.has_value()) { + assert(!resolved_service.value().IsInternal()); } } { diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index 96254aa222..ec3cdbff5a 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -9,6 +9,7 @@ #include <protocol.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> #include <cassert> #include <cstdint> @@ -18,7 +19,7 @@ void initialize_p2p_transport_serialization() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(p2p_transport_serialization, initialize_p2p_transport_serialization) diff --git a/src/test/fuzz/parse_hd_keypath.cpp b/src/test/fuzz/parse_hd_keypath.cpp index 411b70230a..9a2d9a73da 100644 --- a/src/test/fuzz/parse_hd_keypath.cpp +++ b/src/test/fuzz/parse_hd_keypath.cpp @@ -18,6 +18,6 @@ FUZZ_TARGET(parse_hd_keypath) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::vector<uint32_t> random_keypath = ConsumeRandomLengthIntegralVector<uint32_t>(fuzzed_data_provider); - (void)FormatHDKeypath(random_keypath); + (void)FormatHDKeypath(random_keypath, /*apostrophe=*/true); // WriteHDKeypath calls this with false (void)WriteHDKeypath(random_keypath); } diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index 16486f6b96..6d33c1a8cc 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -7,13 +7,14 @@ #include <rpc/client.h> #include <rpc/util.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> #include <limits> #include <string> void initialize_parse_univalue() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(parse_univalue, initialize_parse_univalue) @@ -21,12 +22,9 @@ FUZZ_TARGET_INIT(parse_univalue, initialize_parse_univalue) const std::string random_string(buffer.begin(), buffer.end()); bool valid = true; const UniValue univalue = [&] { - try { - return ParseNonRFCJSONValue(random_string); - } catch (const std::runtime_error&) { - valid = false; - return UniValue{}; - } + UniValue uv; + if (!uv.read(random_string)) valid = false; + return valid ? uv : UniValue{}; }(); if (!valid) { return; diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index e5a3a6e68a..6d584c9f10 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -9,6 +9,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/chaintype.h> #include <util/check.h> #include <util/overflow.h> @@ -19,7 +20,7 @@ void initialize_pow() { - SelectParams(CBaseChainParams::MAIN); + SelectParams(ChainType::MAIN); } FUZZ_TARGET_INIT(pow, initialize_pow) diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 0a7924f226..744ff4701d 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -2,15 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <banman.h> -#include <chainparams.h> #include <consensus/consensus.h> #include <net.h> #include <net_processing.h> +#include <primitives/transaction.h> #include <protocol.h> -#include <scheduler.h> #include <script/script.h> +#include <serialize.h> +#include <span.h> #include <streams.h> +#include <sync.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -19,45 +20,36 @@ #include <test/util/net.h> #include <test/util/setup_common.h> #include <test/util/validation.h> +#include <util/chaintype.h> +#include <util/check.h> +#include <util/time.h> +#include <validation.h> #include <validationinterface.h> #include <version.h> + #include <atomic> -#include <cassert> -#include <chrono> -#include <cstdint> -#include <iosfwd> +#include <cstdlib> #include <iostream> #include <memory> #include <string> +#include <string_view> +#include <vector> namespace { const TestingSetup* g_setup; +std::string_view LIMIT_TO_MESSAGE_TYPE{}; } // namespace -size_t& GetNumMsgTypes() -{ - static size_t g_num_msg_types{0}; - return g_num_msg_types; -} -#define FUZZ_TARGET_MSG(msg_type) \ - struct msg_type##_Count_Before_Main { \ - msg_type##_Count_Before_Main() \ - { \ - ++GetNumMsgTypes(); \ - } \ - } const static g_##msg_type##_count_before_main; \ - FUZZ_TARGET_INIT(process_message_##msg_type, initialize_process_message) \ - { \ - fuzz_target(buffer, #msg_type); \ - } - void initialize_process_message() { - Assert(GetNumMsgTypes() == getAllNetMessageTypes().size()); // If this fails, add or remove the message type below + if (const auto val{std::getenv("LIMIT_TO_MESSAGE_TYPE")}) { + LIMIT_TO_MESSAGE_TYPE = val; + Assert(std::count(getAllNetMessageTypes().begin(), getAllNetMessageTypes().end(), LIMIT_TO_MESSAGE_TYPE)); // Unknown message type passed + } static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>( - /*chain_name=*/CBaseChainParams::REGTEST, + /*chain_type=*/ChainType::REGTEST, /*extra_args=*/{"-txreconciliation"}); g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { @@ -66,7 +58,7 @@ void initialize_process_message() SyncWithValidationInterfaceQueue(); } -void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE) +FUZZ_TARGET_INIT(process_message, initialize_process_message) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -100,40 +92,3 @@ void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE SyncWithValidationInterfaceQueue(); g_setup->m_node.connman->StopNodes(); } - -FUZZ_TARGET_INIT(process_message, initialize_process_message) { fuzz_target(buffer, ""); } -FUZZ_TARGET_MSG(addr); -FUZZ_TARGET_MSG(addrv2); -FUZZ_TARGET_MSG(block); -FUZZ_TARGET_MSG(blocktxn); -FUZZ_TARGET_MSG(cfcheckpt); -FUZZ_TARGET_MSG(cfheaders); -FUZZ_TARGET_MSG(cfilter); -FUZZ_TARGET_MSG(cmpctblock); -FUZZ_TARGET_MSG(feefilter); -FUZZ_TARGET_MSG(filteradd); -FUZZ_TARGET_MSG(filterclear); -FUZZ_TARGET_MSG(filterload); -FUZZ_TARGET_MSG(getaddr); -FUZZ_TARGET_MSG(getblocks); -FUZZ_TARGET_MSG(getblocktxn); -FUZZ_TARGET_MSG(getcfcheckpt); -FUZZ_TARGET_MSG(getcfheaders); -FUZZ_TARGET_MSG(getcfilters); -FUZZ_TARGET_MSG(getdata); -FUZZ_TARGET_MSG(getheaders); -FUZZ_TARGET_MSG(headers); -FUZZ_TARGET_MSG(inv); -FUZZ_TARGET_MSG(mempool); -FUZZ_TARGET_MSG(merkleblock); -FUZZ_TARGET_MSG(notfound); -FUZZ_TARGET_MSG(ping); -FUZZ_TARGET_MSG(pong); -FUZZ_TARGET_MSG(sendaddrv2); -FUZZ_TARGET_MSG(sendcmpct); -FUZZ_TARGET_MSG(sendheaders); -FUZZ_TARGET_MSG(sendtxrcncl); -FUZZ_TARGET_MSG(tx); -FUZZ_TARGET_MSG(verack); -FUZZ_TARGET_MSG(version); -FUZZ_TARGET_MSG(wtxidrelay); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 96339743ba..68d4e02a26 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -24,7 +24,7 @@ const TestingSetup* g_setup; void initialize_process_messages() { static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>( - /*chain_name=*/CBaseChainParams::REGTEST, + /*chain_type=*/ChainType::REGTEST, /*extra_args=*/{"-txreconciliation"}); g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 1c6140c66a..b1858a1800 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -23,6 +23,7 @@ #include <test/util/setup_common.h> #include <tinyformat.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> @@ -37,7 +38,7 @@ namespace { struct RPCFuzzTestingSetup : public TestingSetup { - RPCFuzzTestingSetup(const std::string& chain_name, const std::vector<const char*>& extra_args) : TestingSetup{chain_name, extra_args} + RPCFuzzTestingSetup(const ChainType chain_type, const std::vector<const char*>& extra_args) : TestingSetup{chain_type, extra_args} { } @@ -97,6 +98,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "decoderawtransaction", "decodescript", "deriveaddresses", + "descriptorprocesspsbt", "disconnectnode", "echo", "echojson", @@ -134,6 +136,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "getnetworkinfo", "getnodeaddresses", "getpeerinfo", + "getprioritisedtransactions", "getrawmempool", "getrawtransaction", "getrpcinfo", @@ -363,7 +366,7 @@ FUZZ_TARGET_INIT(rpc, initialize_rpc) try { rpc_testing_setup->CallRPC(rpc_command, arguments); } catch (const UniValue& json_rpc_error) { - const std::string error_msg{find_value(json_rpc_error, "message").get_str()}; + const std::string error_msg{json_rpc_error.find_value("message").get_str()}; // Once c++20 is allowed, starts_with can be used. // if (error_msg.starts_with("Internal bug detected")) { if (0 == error_msg.rfind("Internal bug detected", 0)) { diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 1037dd934a..8a88c1107a 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -22,6 +22,7 @@ #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <univalue.h> +#include <util/chaintype.h> #include <algorithm> #include <cassert> @@ -32,7 +33,7 @@ void initialize_script() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(script, initialize_script) diff --git a/src/test/fuzz/script_format.cpp b/src/test/fuzz/script_format.cpp index 9186746bcf..5aa0ea58ff 100644 --- a/src/test/fuzz/script_format.cpp +++ b/src/test/fuzz/script_format.cpp @@ -11,10 +11,11 @@ #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <univalue.h> +#include <util/chaintype.h> void initialize_script_format() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(script_format, initialize_script_format) diff --git a/src/test/fuzz/script_sigcache.cpp b/src/test/fuzz/script_sigcache.cpp index de895cc69c..f332987987 100644 --- a/src/test/fuzz/script_sigcache.cpp +++ b/src/test/fuzz/script_sigcache.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <chainparamsbase.h> #include <key.h> #include <pubkey.h> #include <script/sigcache.h> diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index c78c22e6cc..8b62daf162 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <chainparamsbase.h> #include <key.h> #include <psbt.h> #include <pubkey.h> @@ -14,6 +13,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/chaintype.h> #include <util/translation.h> #include <cassert> @@ -27,7 +27,7 @@ void initialize_script_sign() { ECC_Start(); - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(script_sign, initialize_script_sign) diff --git a/src/test/fuzz/signet.cpp b/src/test/fuzz/signet.cpp index 303dcf13e3..e9af93c639 100644 --- a/src/test/fuzz/signet.cpp +++ b/src/test/fuzz/signet.cpp @@ -11,6 +11,7 @@ #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <cstdint> #include <optional> @@ -18,7 +19,7 @@ void initialize_signet() { - static const auto testing_setup = MakeNoLogFileContext<>(CBaseChainParams::SIGNET); + static const auto testing_setup = MakeNoLogFileContext<>(ChainType::SIGNET); } FUZZ_TARGET_INIT(signet, initialize_signet) diff --git a/src/test/fuzz/socks5.cpp b/src/test/fuzz/socks5.cpp index 97f643db49..73235b7ced 100644 --- a/src/test/fuzz/socks5.cpp +++ b/src/test/fuzz/socks5.cpp @@ -14,12 +14,12 @@ #include <string> #include <vector> +extern std::chrono::milliseconds g_socks5_recv_timeout; + namespace { -int default_socks5_recv_timeout; +decltype(g_socks5_recv_timeout) default_socks5_recv_timeout; }; -extern int g_socks5_recv_timeout; - void initialize_socks5() { static const auto testing_setup = MakeNoLogFileContext<const BasicTestingSetup>(); @@ -35,7 +35,7 @@ FUZZ_TARGET_INIT(socks5, initialize_socks5) InterruptSocks5(fuzzed_data_provider.ConsumeBool()); // Set FUZZED_SOCKET_FAKE_LATENCY=1 to exercise recv timeout code paths. This // will slow down fuzzing. - g_socks5_recv_timeout = (fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) ? 1 : default_socks5_recv_timeout; + g_socks5_recv_timeout = (fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) ? 1ms : default_socks5_recv_timeout; FuzzedSock fuzzed_sock = ConsumeSock(fuzzed_data_provider); // This Socks5(...) fuzzing harness would have caught CVE-2017-18350 within // a few seconds of fuzzing. diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 5de24a939d..e81efac6e0 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -5,6 +5,8 @@ #include <blockfilter.h> #include <clientversion.h> #include <common/args.h> +#include <common/settings.h> +#include <common/system.h> #include <common/url.h> #include <netbase.h> #include <outputtype.h> @@ -21,10 +23,8 @@ #include <test/fuzz/util.h> #include <util/error.h> #include <util/fees.h> -#include <util/settings.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <cassert> @@ -63,13 +63,9 @@ FUZZ_TARGET(string) (void)IsDeprecatedRPCEnabled(random_string_1); (void)Join(random_string_vector, random_string_1); (void)JSONRPCError(fuzzed_data_provider.ConsumeIntegral<int>(), random_string_1); - const util::Settings settings; + const common::Settings settings; (void)OnlyHasDefaultSectionSetting(settings, random_string_1, random_string_2); (void)ParseNetwork(random_string_1); - try { - (void)ParseNonRFCJSONValue(random_string_1); - } catch (const std::runtime_error&) { - } (void)ParseOutputType(random_string_1); (void)RemovePrefix(random_string_1, random_string_2); (void)ResolveErrMsg(random_string_1, random_string_2); diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index 935f0c21e1..73c01d9297 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -55,7 +55,7 @@ FUZZ_TARGET_INIT(system, initialize_system) [&] { const OptionsCategory options_category = fuzzed_data_provider.PickValueInArray<OptionsCategory>({OptionsCategory::OPTIONS, OptionsCategory::CONNECTION, OptionsCategory::WALLET, OptionsCategory::WALLET_DEBUG_TEST, OptionsCategory::ZMQ, OptionsCategory::DEBUG_TEST, OptionsCategory::CHAINPARAMS, OptionsCategory::NODE_RELAY, OptionsCategory::BLOCK_CREATION, OptionsCategory::RPC, OptionsCategory::GUI, OptionsCategory::COMMANDS, OptionsCategory::REGISTER_COMMANDS, OptionsCategory::HIDDEN}); // Avoid hitting: - // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + // common/args.cpp:563: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. const std::string argument_name = GetArgumentName(fuzzed_data_provider.ConsumeRandomLengthString(16)); if (args_manager.GetArgFlags(argument_name) != std::nullopt) { return; @@ -64,7 +64,7 @@ FUZZ_TARGET_INIT(system, initialize_system) }, [&] { // Avoid hitting: - // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + // common/args.cpp:563: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. const std::vector<std::string> names = ConsumeRandomLengthStringVector(fuzzed_data_provider); std::vector<std::string> hidden_arguments; for (const std::string& name : names) { @@ -108,7 +108,7 @@ FUZZ_TARGET_INIT(system, initialize_system) (void)args_manager.GetArgs(s1); (void)args_manager.GetBoolArg(s1, b); try { - (void)args_manager.GetChainName(); + (void)args_manager.GetChainTypeString(); } catch (const std::runtime_error&) { } (void)args_manager.GetHelpMessage(); diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index bacb178b44..7035c53d13 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -15,6 +15,7 @@ #include <streams.h> #include <test/fuzz/fuzz.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/rbf.h> #include <validation.h> #include <version.h> @@ -23,7 +24,7 @@ void initialize_transaction() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(transaction, initialize_transaction) diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index a055705575..b758c715ef 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -42,12 +42,12 @@ void initialize_tx_pool() g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; ++i) { - CTxIn in = MineBlock(g_setup->m_node, P2WSH_OP_TRUE); + COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_OP_TRUE)}; // Remember the txids to avoid expensive disk access later on auto& outpoints = i < COINBASE_MATURITY ? g_outpoints_coinbase_init_mature : g_outpoints_coinbase_init_immature; - outpoints.push_back(in.prevout); + outpoints.push_back(prevout); } SyncWithValidationInterfaceQueue(); } diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index c14f633029..5f2aff08da 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -6,7 +6,6 @@ #define BITCOIN_TEST_FUZZ_UTIL_H #include <arith_uint256.h> -#include <chainparamsbase.h> #include <coins.h> #include <compat/compat.h> #include <consensus/amount.h> diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index be34906025..b4ef0c7a3e 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -10,6 +10,7 @@ #include <test/fuzz/util.h> #include <test/util/mining.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <util/fs.h> #include <validation.h> #include <validationinterface.h> @@ -22,7 +23,7 @@ const std::vector<std::shared_ptr<CBlock>>* g_chain; void initialize_chain() { - const auto params{CreateChainParams(ArgsManager{}, CBaseChainParams::REGTEST)}; + const auto params{CreateChainParams(ArgsManager{}, ChainType::REGTEST)}; static const auto chain{CreateBlockChain(2 * COINBASE_MATURITY, *params)}; g_chain = &chain; } diff --git a/src/test/fuzz/utxo_total_supply.cpp b/src/test/fuzz/utxo_total_supply.cpp new file mode 100644 index 0000000000..318797faf2 --- /dev/null +++ b/src/test/fuzz/utxo_total_supply.cpp @@ -0,0 +1,168 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <consensus/consensus.h> +#include <consensus/merkle.h> +#include <kernel/coinstats.h> +#include <node/miner.h> +#include <script/interpreter.h> +#include <streams.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/mining.h> +#include <test/util/setup_common.h> +#include <util/chaintype.h> +#include <validation.h> +#include <version.h> + +FUZZ_TARGET(utxo_total_supply) +{ + /** The testing setup that creates a chainman only (no chainstate) */ + ChainTestingSetup test_setup{ + ChainType::REGTEST, + { + "-testactivationheight=bip34@2", + }, + }; + // Create chainstate + test_setup.LoadVerifyActivateChainstate(); + auto& node{test_setup.m_node}; + auto& chainman{*Assert(test_setup.m_node.chainman)}; + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const auto ActiveHeight = [&]() { + LOCK(chainman.GetMutex()); + return chainman.ActiveHeight(); + }; + const auto PrepareNextBlock = [&]() { + // Use OP_FALSE to avoid BIP30 check from hitting early + auto block = PrepareBlock(node, CScript{} << OP_FALSE); + // Replace OP_FALSE with OP_TRUE + { + CMutableTransaction tx{*block->vtx.back()}; + tx.vout.at(0).scriptPubKey = CScript{} << OP_TRUE; + block->vtx.back() = MakeTransactionRef(tx); + } + return block; + }; + + /** The block template this fuzzer is working on */ + auto current_block = PrepareNextBlock(); + /** Append-only set of tx outpoints, entries are not removed when spent */ + std::vector<std::pair<COutPoint, CTxOut>> txos; + /** The utxo stats at the chain tip */ + kernel::CCoinsStats utxo_stats; + /** The total amount of coins in the utxo set */ + CAmount circulation{0}; + + + // Store the tx out in the txo map + const auto StoreLastTxo = [&]() { + // get last tx + const CTransaction& tx = *current_block->vtx.back(); + // get last out + const uint32_t i = tx.vout.size() - 1; + // store it + txos.emplace_back(COutPoint{tx.GetHash(), i}, tx.vout.at(i)); + if (current_block->vtx.size() == 1 && tx.vout.at(i).scriptPubKey[0] == OP_RETURN) { + // also store coinbase + const uint32_t i = tx.vout.size() - 2; + txos.emplace_back(COutPoint{tx.GetHash(), i}, tx.vout.at(i)); + } + }; + const auto AppendRandomTxo = [&](CMutableTransaction& tx) { + const auto& txo = txos.at(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, txos.size() - 1)); + tx.vin.emplace_back(txo.first); + tx.vout.emplace_back(txo.second.nValue, txo.second.scriptPubKey); // "Forward" coin with no fee + }; + const auto UpdateUtxoStats = [&]() { + LOCK(chainman.GetMutex()); + chainman.ActiveChainstate().ForceFlushStateToDisk(); + utxo_stats = std::move( + *Assert(kernel::ComputeUTXOStats(kernel::CoinStatsHashType::NONE, &chainman.ActiveChainstate().CoinsDB(), chainman.m_blockman, {}))); + // Check that miner can't print more money than they are allowed to + assert(circulation == utxo_stats.total_amount); + }; + + + // Update internal state to chain tip + StoreLastTxo(); + UpdateUtxoStats(); + assert(ActiveHeight() == 0); + // Get at which height we duplicate the coinbase + // Assuming that the fuzzer will mine relatively short chains (less than 200 blocks), we want the duplicate coinbase to be not too high. + // Up to 2000 seems reasonable. + int64_t duplicate_coinbase_height = fuzzed_data_provider.ConsumeIntegralInRange(0, 20 * COINBASE_MATURITY); + // Always pad with OP_0 at the end to avoid bad-cb-length error + const CScript duplicate_coinbase_script = CScript() << duplicate_coinbase_height << OP_0; + // Mine the first block with this duplicate + current_block = PrepareNextBlock(); + StoreLastTxo(); + + { + // Create duplicate (CScript should match exact format as in CreateNewBlock) + CMutableTransaction tx{*current_block->vtx.front()}; + tx.vin.at(0).scriptSig = duplicate_coinbase_script; + + // Mine block and create next block template + current_block->vtx.front() = MakeTransactionRef(tx); + } + current_block->hashMerkleRoot = BlockMerkleRoot(*current_block); + assert(!MineBlock(node, current_block).IsNull()); + circulation += GetBlockSubsidy(ActiveHeight(), Params().GetConsensus()); + + assert(ActiveHeight() == 1); + UpdateUtxoStats(); + current_block = PrepareNextBlock(); + StoreLastTxo(); + + // Limit to avoid timeout, but enough to cover duplicate_coinbase_height + // and CVE-2018-17144. + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'000) + { + CallOneOf( + fuzzed_data_provider, + [&] { + // Append an input-output pair to the last tx in the current block + CMutableTransaction tx{*current_block->vtx.back()}; + AppendRandomTxo(tx); + current_block->vtx.back() = MakeTransactionRef(tx); + StoreLastTxo(); + }, + [&] { + // Append a tx to the list of txs in the current block + CMutableTransaction tx{}; + AppendRandomTxo(tx); + current_block->vtx.push_back(MakeTransactionRef(tx)); + StoreLastTxo(); + }, + [&] { + // Append the current block to the active chain + node::RegenerateCommitments(*current_block, chainman); + const bool was_valid = !MineBlock(node, current_block).IsNull(); + + const auto prev_utxo_stats = utxo_stats; + if (was_valid) { + if (duplicate_coinbase_height == ActiveHeight()) { + // we mined the duplicate coinbase + assert(current_block->vtx.at(0)->vin.at(0).scriptSig == duplicate_coinbase_script); + } + + circulation += GetBlockSubsidy(ActiveHeight(), Params().GetConsensus()); + } + + UpdateUtxoStats(); + + if (!was_valid) { + // utxo stats must not change + assert(prev_utxo_stats.hashSerialized == utxo_stats.hashSerialized); + } + + current_block = PrepareNextBlock(); + StoreLastTxo(); + }); + } +} diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 817593cf6e..4e2ca6d903 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -4,7 +4,6 @@ #include <kernel/mempool_persist.h> -#include <chainparamsbase.h> #include <node/mempool_args.h> #include <node/mempool_persist_args.h> #include <test/fuzz/FuzzedDataProvider.h> diff --git a/src/test/fuzz/versionbits.cpp b/src/test/fuzz/versionbits.cpp index e6a19d6e91..b3df4dadd2 100644 --- a/src/test/fuzz/versionbits.cpp +++ b/src/test/fuzz/versionbits.cpp @@ -7,6 +7,7 @@ #include <common/args.h> #include <consensus/params.h> #include <primitives/block.h> +#include <util/chaintype.h> #include <versionbits.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -104,7 +105,7 @@ std::unique_ptr<const CChainParams> g_params; void initialize() { // this is actually comparatively slow, so only do it once - g_params = CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN); + g_params = CreateChainParams(ArgsManager{}, ChainType::MAIN); assert(g_params != nullptr); } diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 715b6885f5..c73b675388 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -3,10 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <common/args.h> +#include <common/settings.h> #include <logging.h> #include <test/util/setup_common.h> #include <univalue.h> -#include <util/settings.h> #include <util/strencodings.h> #include <limits> @@ -57,8 +57,8 @@ BOOST_AUTO_TEST_CASE(setting_args) ArgsManager args; SetupArgs(args, {{"-foo", ArgsManager::ALLOW_ANY}}); - auto set_foo = [&](const util::SettingsValue& value) { - args.LockSettings([&](util::Settings& settings) { + auto set_foo = [&](const common::SettingsValue& value) { + args.LockSettings([&](common::Settings& settings) { settings.rw_settings["foo"] = value; }); }; diff --git a/src/test/interfaces_tests.cpp b/src/test/interfaces_tests.cpp index d37fe35a11..68377e197f 100644 --- a/src/test/interfaces_tests.cpp +++ b/src/test/interfaces_tests.cpp @@ -98,7 +98,7 @@ BOOST_AUTO_TEST_CASE(findAncestorByHash) BOOST_AUTO_TEST_CASE(findCommonAncestor) { auto& chain = m_node.chain; - const CChain& active = WITH_LOCK(Assert(m_node.chainman)->GetMutex(), return Assert(m_node.chainman)->ActiveChain()); + const CChain& active{*WITH_LOCK(Assert(m_node.chainman)->GetMutex(), return &Assert(m_node.chainman)->ActiveChain())}; auto* orig_tip = active.Tip(); for (int i = 0; i < 10; ++i) { BlockValidationState state; diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index a400afee71..92bdbae3c4 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -10,6 +10,7 @@ #include <script/script.h> #include <test/util/json.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <boost/test/unit_test.hpp> @@ -24,7 +25,7 @@ BOOST_AUTO_TEST_CASE(key_io_valid_parse) UniValue tests = read_json(std::string(json_tests::key_io_valid, json_tests::key_io_valid + sizeof(json_tests::key_io_valid))); CKey privkey; CTxDestination destination; - SelectParams(CBaseChainParams::MAIN); + SelectParams(ChainType::MAIN); for (unsigned int idx = 0; idx < tests.size(); idx++) { const UniValue& test = tests[idx]; @@ -36,11 +37,11 @@ BOOST_AUTO_TEST_CASE(key_io_valid_parse) std::string exp_base58string = test[0].get_str(); const std::vector<std::byte> exp_payload{ParseHex<std::byte>(test[1].get_str())}; const UniValue &metadata = test[2].get_obj(); - bool isPrivkey = find_value(metadata, "isPrivkey").get_bool(); - SelectParams(find_value(metadata, "chain").get_str()); - bool try_case_flip = find_value(metadata, "tryCaseFlip").isNull() ? false : find_value(metadata, "tryCaseFlip").get_bool(); + bool isPrivkey = metadata.find_value("isPrivkey").get_bool(); + SelectParams(ChainTypeFromString(metadata.find_value("chain").get_str()).value()); + bool try_case_flip = metadata.find_value("tryCaseFlip").isNull() ? false : metadata.find_value("tryCaseFlip").get_bool(); if (isPrivkey) { - bool isCompressed = find_value(metadata, "isCompressed").get_bool(); + bool isCompressed = metadata.find_value("isCompressed").get_bool(); // Must be valid private key privkey = DecodeSecret(exp_base58string); BOOST_CHECK_MESSAGE(privkey.IsValid(), "!IsValid:" + strTest); @@ -95,10 +96,10 @@ BOOST_AUTO_TEST_CASE(key_io_valid_gen) std::string exp_base58string = test[0].get_str(); std::vector<unsigned char> exp_payload = ParseHex(test[1].get_str()); const UniValue &metadata = test[2].get_obj(); - bool isPrivkey = find_value(metadata, "isPrivkey").get_bool(); - SelectParams(find_value(metadata, "chain").get_str()); + bool isPrivkey = metadata.find_value("isPrivkey").get_bool(); + SelectParams(ChainTypeFromString(metadata.find_value("chain").get_str()).value()); if (isPrivkey) { - bool isCompressed = find_value(metadata, "isCompressed").get_bool(); + bool isCompressed = metadata.find_value("isCompressed").get_bool(); CKey key; key.Set(exp_payload.begin(), exp_payload.end(), isCompressed); assert(key.IsValid()); @@ -113,7 +114,7 @@ BOOST_AUTO_TEST_CASE(key_io_valid_gen) } } - SelectParams(CBaseChainParams::MAIN); + SelectParams(ChainType::MAIN); } @@ -135,7 +136,7 @@ BOOST_AUTO_TEST_CASE(key_io_invalid) std::string exp_base58string = test[0].get_str(); // must be invalid as public and as private key - for (const auto& chain : { CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET, CBaseChainParams::REGTEST }) { + for (const auto& chain : {ChainType::MAIN, ChainType::TESTNET, ChainType::SIGNET, ChainType::REGTEST}) { SelectParams(chain); destination = DecodeDestination(exp_base58string); BOOST_CHECK_MESSAGE(!IsValidDestination(destination), "IsValid pubkey in mainnet:" + strTest); diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index ea5b94f3a5..8f11bf5db2 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -4,6 +4,7 @@ #include <key.h> +#include <common/system.h> #include <key_io.h> #include <streams.h> #include <test/util/random.h> @@ -11,7 +12,6 @@ #include <uint256.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <string> #include <vector> diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 94e553a304..db58a0baec 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <policy/policy.h> #include <test/util/txmempool.h> #include <txmempool.h> -#include <util/system.h> #include <util/time.h> #include <test/util/setup_common.h> diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index cfab762307..94e3f27930 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <coins.h> +#include <common/system.h> #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_verify.h> @@ -15,7 +16,6 @@ #include <txmempool.h> #include <uint256.h> #include <util/strencodings.h> -#include <util/system.h> #include <util/time.h> #include <validation.h> #include <versionbits.h> diff --git a/src/test/miniminer_tests.cpp b/src/test/miniminer_tests.cpp new file mode 100644 index 0000000000..b26f7dafe3 --- /dev/null +++ b/src/test/miniminer_tests.cpp @@ -0,0 +1,476 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <node/mini_miner.h> +#include <txmempool.h> +#include <util/time.h> + +#include <test/util/setup_common.h> +#include <test/util/txmempool.h> + +#include <boost/test/unit_test.hpp> +#include <optional> +#include <vector> + +BOOST_FIXTURE_TEST_SUITE(miniminer_tests, TestingSetup) + +static inline CTransactionRef make_tx(const std::vector<COutPoint>& inputs, size_t num_outputs) +{ + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(inputs.size()); + tx.vout.resize(num_outputs); + for (size_t i = 0; i < inputs.size(); ++i) { + tx.vin[i].prevout = inputs[i]; + } + for (size_t i = 0; i < num_outputs; ++i) { + tx.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + // The actual input and output values of these transactions don't really + // matter, since all accounting will use the entries' cached fees. + tx.vout[i].nValue = COIN; + } + return MakeTransactionRef(tx); +} + +static inline bool sanity_check(const std::vector<CTransactionRef>& transactions, + const std::map<COutPoint, CAmount>& bumpfees) +{ + // No negative bumpfees. + for (const auto& [outpoint, fee] : bumpfees) { + if (fee < 0) return false; + if (fee == 0) continue; + auto outpoint_ = outpoint; // structured bindings can't be captured in C++17, so we need to use a variable + const bool found = std::any_of(transactions.cbegin(), transactions.cend(), [&](const auto& tx) { + return outpoint_.hash == tx->GetHash() && outpoint_.n < tx->vout.size(); + }); + if (!found) return false; + } + for (const auto& tx : transactions) { + // If tx has multiple outputs, they must all have the same bumpfee (if they exist). + if (tx->vout.size() > 1) { + std::set<CAmount> distinct_bumpfees; + for (size_t i{0}; i < tx->vout.size(); ++i) { + const auto bumpfee = bumpfees.find(COutPoint{tx->GetHash(), static_cast<uint32_t>(i)}); + if (bumpfee != bumpfees.end()) distinct_bumpfees.insert(bumpfee->second); + } + if (distinct_bumpfees.size() > 1) return false; + } + } + return true; +} + +template <typename Key, typename Value> +Value Find(const std::map<Key, Value>& map, const Key& key) +{ + auto it = map.find(key); + BOOST_CHECK_MESSAGE(it != map.end(), strprintf("Cannot find %s", key.ToString())); + return it->second; +} + +BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; + + const CAmount low_fee{CENT/2000}; + const CAmount normal_fee{CENT/200}; + const CAmount high_fee{CENT/10}; + + // Create a parent tx1 and child tx2 with normal fees: + const auto tx1 = make_tx({COutPoint{m_coinbase_txns[0]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx1)); + const auto tx2 = make_tx({COutPoint{tx1->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx2)); + + // Create a low-feerate parent tx3 and high-feerate child tx4 (cpfp) + const auto tx3 = make_tx({COutPoint{m_coinbase_txns[1]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx3)); + const auto tx4 = make_tx({COutPoint{tx3->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx4)); + + // Create a parent tx5 and child tx6 where both have low fees + const auto tx5 = make_tx({COutPoint{m_coinbase_txns[2]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx5)); + const auto tx6 = make_tx({COutPoint{tx5->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx6)); + // Make tx6's modified fee much higher than its base fee. This should cause it to pass + // the fee-related checks despite being low-feerate. + pool.PrioritiseTransaction(tx6->GetHash(), CENT/100); + + // Create a high-feerate parent tx7, low-feerate child tx8 + const auto tx7 = make_tx({COutPoint{m_coinbase_txns[3]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx7)); + const auto tx8 = make_tx({COutPoint{tx7->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx8)); + + std::vector<COutPoint> all_unspent_outpoints({ + COutPoint{tx1->GetHash(), 1}, + COutPoint{tx2->GetHash(), 0}, + COutPoint{tx3->GetHash(), 1}, + COutPoint{tx4->GetHash(), 0}, + COutPoint{tx5->GetHash(), 1}, + COutPoint{tx6->GetHash(), 0}, + COutPoint{tx7->GetHash(), 1}, + COutPoint{tx8->GetHash(), 0} + }); + for (const auto& outpoint : all_unspent_outpoints) BOOST_CHECK(!pool.isSpent(outpoint)); + + std::vector<COutPoint> all_spent_outpoints({ + COutPoint{tx1->GetHash(), 0}, + COutPoint{tx3->GetHash(), 0}, + COutPoint{tx5->GetHash(), 0}, + COutPoint{tx7->GetHash(), 0} + }); + for (const auto& outpoint : all_spent_outpoints) BOOST_CHECK(pool.GetConflictTx(outpoint) != nullptr); + + std::vector<COutPoint> all_parent_outputs({ + COutPoint{tx1->GetHash(), 0}, + COutPoint{tx1->GetHash(), 1}, + COutPoint{tx3->GetHash(), 0}, + COutPoint{tx3->GetHash(), 1}, + COutPoint{tx5->GetHash(), 0}, + COutPoint{tx5->GetHash(), 1}, + COutPoint{tx7->GetHash(), 0}, + COutPoint{tx7->GetHash(), 1} + }); + + + std::vector<CTransactionRef> all_transactions{tx1, tx2, tx3, tx4, tx5, tx6, tx7, tx8}; + struct TxDimensions { + int32_t vsize; CAmount mod_fee; CFeeRate feerate; + }; + std::map<uint256, TxDimensions> tx_dims; + for (const auto& tx : all_transactions) { + const auto it = pool.GetIter(tx->GetHash()).value(); + tx_dims.emplace(tx->GetHash(), TxDimensions{it->GetTxSize(), it->GetModifiedFee(), + CFeeRate(it->GetModifiedFee(), it->GetTxSize())}); + } + + const std::vector<CFeeRate> various_normal_feerates({CFeeRate(0), CFeeRate(500), CFeeRate(999), + CFeeRate(1000), CFeeRate(2000), CFeeRate(2500), + CFeeRate(3333), CFeeRate(7800), CFeeRate(11199), + CFeeRate(23330), CFeeRate(50000), CFeeRate(5*CENT)}); + + // All nonexistent entries have a bumpfee of zero, regardless of feerate + std::vector<COutPoint> nonexistent_outpoints({ COutPoint{GetRandHash(), 0}, COutPoint{GetRandHash(), 3} }); + for (const auto& outpoint : nonexistent_outpoints) BOOST_CHECK(!pool.isSpent(outpoint)); + for (const auto& feerate : various_normal_feerates) { + node::MiniMiner mini_miner(pool, nonexistent_outpoints); + BOOST_CHECK(mini_miner.IsReadyToCalculate()); + auto bump_fees = mini_miner.CalculateBumpFees(feerate); + BOOST_CHECK(!mini_miner.IsReadyToCalculate()); + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + BOOST_CHECK(bump_fees.size() == nonexistent_outpoints.size()); + for (const auto& outpoint: nonexistent_outpoints) { + auto it = bump_fees.find(outpoint); + BOOST_CHECK(it != bump_fees.end()); + BOOST_CHECK_EQUAL(it->second, 0); + } + } + + // Gather bump fees for all available UTXOs. + for (const auto& target_feerate : various_normal_feerates) { + node::MiniMiner mini_miner(pool, all_unspent_outpoints); + BOOST_CHECK(mini_miner.IsReadyToCalculate()); + auto bump_fees = mini_miner.CalculateBumpFees(target_feerate); + BOOST_CHECK(!mini_miner.IsReadyToCalculate()); + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + BOOST_CHECK_EQUAL(bump_fees.size(), all_unspent_outpoints.size()); + + // Check tx1 bumpfee: no other bumper. + const TxDimensions& tx1_dimensions = tx_dims.find(tx1->GetHash())->second; + CAmount bumpfee1 = Find(bump_fees, COutPoint{tx1->GetHash(), 1}); + if (target_feerate <= tx1_dimensions.feerate) { + BOOST_CHECK_EQUAL(bumpfee1, 0); + } else { + // Difference is fee to bump tx1 from current to target feerate. + BOOST_CHECK_EQUAL(bumpfee1, target_feerate.GetFee(tx1_dimensions.vsize) - tx1_dimensions.mod_fee); + } + + // Check tx3 bumpfee: assisted by tx4. + const TxDimensions& tx3_dimensions = tx_dims.find(tx3->GetHash())->second; + const TxDimensions& tx4_dimensions = tx_dims.find(tx4->GetHash())->second; + const CFeeRate tx3_feerate = CFeeRate(tx3_dimensions.mod_fee + tx4_dimensions.mod_fee, tx3_dimensions.vsize + tx4_dimensions.vsize); + CAmount bumpfee3 = Find(bump_fees, COutPoint{tx3->GetHash(), 1}); + if (target_feerate <= tx3_feerate) { + // As long as target feerate is below tx4's ancestor feerate, there is no bump fee. + BOOST_CHECK_EQUAL(bumpfee3, 0); + } else { + // Difference is fee to bump tx3 from current to target feerate, without tx4. + BOOST_CHECK_EQUAL(bumpfee3, target_feerate.GetFee(tx3_dimensions.vsize) - tx3_dimensions.mod_fee); + } + + // If tx6’s modified fees are sufficient for tx5 and tx6 to be picked + // into the block, our prospective new transaction would not need to + // bump tx5 when using tx5’s second output. If however even tx6’s + // modified fee (which essentially indicates "effective feerate") is + // not sufficient to bump tx5, using the second output of tx5 would + // require our transaction to bump tx5 from scratch since we evaluate + // transaction packages per ancestor sets and do not consider multiple + // children’s fees. + const TxDimensions& tx5_dimensions = tx_dims.find(tx5->GetHash())->second; + const TxDimensions& tx6_dimensions = tx_dims.find(tx6->GetHash())->second; + const CFeeRate tx5_feerate = CFeeRate(tx5_dimensions.mod_fee + tx6_dimensions.mod_fee, tx5_dimensions.vsize + tx6_dimensions.vsize); + CAmount bumpfee5 = Find(bump_fees, COutPoint{tx5->GetHash(), 1}); + if (target_feerate <= tx5_feerate) { + // As long as target feerate is below tx6's ancestor feerate, there is no bump fee. + BOOST_CHECK_EQUAL(bumpfee5, 0); + } else { + // Difference is fee to bump tx5 from current to target feerate, without tx6. + BOOST_CHECK_EQUAL(bumpfee5, target_feerate.GetFee(tx5_dimensions.vsize) - tx5_dimensions.mod_fee); + } + } + // Spent outpoints should usually not be requested as they would not be + // considered available. However, when they are explicitly requested, we + // can calculate their bumpfee to facilitate RBF-replacements + for (const auto& target_feerate : various_normal_feerates) { + node::MiniMiner mini_miner_all_spent(pool, all_spent_outpoints); + BOOST_CHECK(mini_miner_all_spent.IsReadyToCalculate()); + auto bump_fees_all_spent = mini_miner_all_spent.CalculateBumpFees(target_feerate); + BOOST_CHECK(!mini_miner_all_spent.IsReadyToCalculate()); + BOOST_CHECK_EQUAL(bump_fees_all_spent.size(), all_spent_outpoints.size()); + node::MiniMiner mini_miner_all_parents(pool, all_parent_outputs); + BOOST_CHECK(mini_miner_all_parents.IsReadyToCalculate()); + auto bump_fees_all_parents = mini_miner_all_parents.CalculateBumpFees(target_feerate); + BOOST_CHECK(!mini_miner_all_parents.IsReadyToCalculate()); + BOOST_CHECK_EQUAL(bump_fees_all_parents.size(), all_parent_outputs.size()); + for (auto& bump_fees : {bump_fees_all_parents, bump_fees_all_spent}) { + // For all_parents case, both outputs from the parent should have the same bump fee, + // even though only one of them is in a to-be-replaced transaction. + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + + // Check tx1 bumpfee: no other bumper. + const TxDimensions& tx1_dimensions = tx_dims.find(tx1->GetHash())->second; + CAmount it1_spent = Find(bump_fees, COutPoint{tx1->GetHash(), 0}); + if (target_feerate <= tx1_dimensions.feerate) { + BOOST_CHECK_EQUAL(it1_spent, 0); + } else { + // Difference is fee to bump tx1 from current to target feerate. + BOOST_CHECK_EQUAL(it1_spent, target_feerate.GetFee(tx1_dimensions.vsize) - tx1_dimensions.mod_fee); + } + + // Check tx3 bumpfee: no other bumper, because tx4 is to-be-replaced. + const TxDimensions& tx3_dimensions = tx_dims.find(tx3->GetHash())->second; + const CFeeRate tx3_feerate_unbumped = tx3_dimensions.feerate; + auto it3_spent = Find(bump_fees, COutPoint{tx3->GetHash(), 0}); + if (target_feerate <= tx3_feerate_unbumped) { + BOOST_CHECK_EQUAL(it3_spent, 0); + } else { + // Difference is fee to bump tx3 from current to target feerate, without tx4. + BOOST_CHECK_EQUAL(it3_spent, target_feerate.GetFee(tx3_dimensions.vsize) - tx3_dimensions.mod_fee); + } + + // Check tx5 bumpfee: no other bumper, because tx6 is to-be-replaced. + const TxDimensions& tx5_dimensions = tx_dims.find(tx5->GetHash())->second; + const CFeeRate tx5_feerate_unbumped = tx5_dimensions.feerate; + auto it5_spent = Find(bump_fees, COutPoint{tx5->GetHash(), 0}); + if (target_feerate <= tx5_feerate_unbumped) { + BOOST_CHECK_EQUAL(it5_spent, 0); + } else { + // Difference is fee to bump tx5 from current to target feerate, without tx6. + BOOST_CHECK_EQUAL(it5_spent, target_feerate.GetFee(tx5_dimensions.vsize) - tx5_dimensions.mod_fee); + } + } + } +} + +BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; + + const CAmount low_fee{CENT/2000}; + const CAmount med_fee{CENT/200}; + const CAmount high_fee{CENT/10}; + + // Create 3 parents of different feerates, and 1 child spending from all 3. + const auto tx1 = make_tx({COutPoint{m_coinbase_txns[0]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx1)); + const auto tx2 = make_tx({COutPoint{m_coinbase_txns[1]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(med_fee).FromTx(tx2)); + const auto tx3 = make_tx({COutPoint{m_coinbase_txns[2]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx3)); + const auto tx4 = make_tx({COutPoint{tx1->GetHash(), 0}, COutPoint{tx2->GetHash(), 0}, COutPoint{tx3->GetHash(), 0}}, /*num_outputs=*/3); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx4)); + + // Create 1 grandparent and 1 parent, then 2 children. + const auto tx5 = make_tx({COutPoint{m_coinbase_txns[3]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx5)); + const auto tx6 = make_tx({COutPoint{tx5->GetHash(), 0}}, /*num_outputs=*/3); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx6)); + const auto tx7 = make_tx({COutPoint{tx6->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(med_fee).FromTx(tx7)); + const auto tx8 = make_tx({COutPoint{tx6->GetHash(), 1}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx8)); + + std::vector<CTransactionRef> all_transactions{tx1, tx2, tx3, tx4, tx5, tx6, tx7, tx8}; + std::vector<int64_t> tx_vsizes; + tx_vsizes.reserve(all_transactions.size()); + for (const auto& tx : all_transactions) tx_vsizes.push_back(GetVirtualTransactionSize(*tx)); + + std::vector<COutPoint> all_unspent_outpoints({ + COutPoint{tx1->GetHash(), 1}, + COutPoint{tx2->GetHash(), 1}, + COutPoint{tx3->GetHash(), 1}, + COutPoint{tx4->GetHash(), 0}, + COutPoint{tx4->GetHash(), 1}, + COutPoint{tx4->GetHash(), 2}, + COutPoint{tx5->GetHash(), 1}, + COutPoint{tx6->GetHash(), 2}, + COutPoint{tx7->GetHash(), 0}, + COutPoint{tx8->GetHash(), 0} + }); + for (const auto& outpoint : all_unspent_outpoints) BOOST_CHECK(!pool.isSpent(outpoint)); + + const auto tx3_feerate = CFeeRate(high_fee, tx_vsizes[2]); + const auto tx4_feerate = CFeeRate(high_fee, tx_vsizes[3]); + // tx4's feerate is lower than tx3's. same fee, different weight. + BOOST_CHECK(tx3_feerate > tx4_feerate); + const auto tx4_anc_feerate = CFeeRate(low_fee + med_fee + high_fee, tx_vsizes[0] + tx_vsizes[1] + tx_vsizes[3]); + const auto tx5_feerate = CFeeRate(high_fee, tx_vsizes[4]); + const auto tx7_anc_feerate = CFeeRate(low_fee + med_fee, tx_vsizes[5] + tx_vsizes[6]); + const auto tx8_anc_feerate = CFeeRate(low_fee + high_fee, tx_vsizes[5] + tx_vsizes[7]); + BOOST_CHECK(tx5_feerate > tx7_anc_feerate); + BOOST_CHECK(tx5_feerate > tx8_anc_feerate); + + // Extremely high feerate: everybody's bumpfee is from their full ancestor set. + { + node::MiniMiner mini_miner(pool, all_unspent_outpoints); + const CFeeRate very_high_feerate(COIN); + BOOST_CHECK(tx4_anc_feerate < very_high_feerate); + BOOST_CHECK(mini_miner.IsReadyToCalculate()); + auto bump_fees = mini_miner.CalculateBumpFees(very_high_feerate); + BOOST_CHECK_EQUAL(bump_fees.size(), all_unspent_outpoints.size()); + BOOST_CHECK(!mini_miner.IsReadyToCalculate()); + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + const auto tx1_bumpfee = bump_fees.find(COutPoint{tx1->GetHash(), 1}); + BOOST_CHECK(tx1_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx1_bumpfee->second, very_high_feerate.GetFee(tx_vsizes[0]) - low_fee); + const auto tx4_bumpfee = bump_fees.find(COutPoint{tx4->GetHash(), 0}); + BOOST_CHECK(tx4_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx4_bumpfee->second, + very_high_feerate.GetFee(tx_vsizes[0] + tx_vsizes[1] + tx_vsizes[2] + tx_vsizes[3]) - (low_fee + med_fee + high_fee + high_fee)); + const auto tx7_bumpfee = bump_fees.find(COutPoint{tx7->GetHash(), 0}); + BOOST_CHECK(tx7_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx7_bumpfee->second, + very_high_feerate.GetFee(tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[6]) - (high_fee + low_fee + med_fee)); + const auto tx8_bumpfee = bump_fees.find(COutPoint{tx8->GetHash(), 0}); + BOOST_CHECK(tx8_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx8_bumpfee->second, + very_high_feerate.GetFee(tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[7]) - (high_fee + low_fee + high_fee)); + // Total fees: if spending multiple outputs from tx4 don't double-count fees. + node::MiniMiner mini_miner_total_tx4(pool, {COutPoint{tx4->GetHash(), 0}, COutPoint{tx4->GetHash(), 1}}); + BOOST_CHECK(mini_miner_total_tx4.IsReadyToCalculate()); + const auto tx4_bump_fee = mini_miner_total_tx4.CalculateTotalBumpFees(very_high_feerate); + BOOST_CHECK(!mini_miner_total_tx4.IsReadyToCalculate()); + BOOST_CHECK(tx4_bump_fee.has_value()); + BOOST_CHECK_EQUAL(tx4_bump_fee.value(), + very_high_feerate.GetFee(tx_vsizes[0] + tx_vsizes[1] + tx_vsizes[2] + tx_vsizes[3]) - (low_fee + med_fee + high_fee + high_fee)); + // Total fees: if spending both tx7 and tx8, don't double-count fees. + node::MiniMiner mini_miner_tx7_tx8(pool, {COutPoint{tx7->GetHash(), 0}, COutPoint{tx8->GetHash(), 0}}); + BOOST_CHECK(mini_miner_tx7_tx8.IsReadyToCalculate()); + const auto tx7_tx8_bumpfee = mini_miner_tx7_tx8.CalculateTotalBumpFees(very_high_feerate); + BOOST_CHECK(!mini_miner_tx7_tx8.IsReadyToCalculate()); + BOOST_CHECK(tx7_tx8_bumpfee.has_value()); + BOOST_CHECK_EQUAL(tx7_tx8_bumpfee.value(), + very_high_feerate.GetFee(tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[6] + tx_vsizes[7]) - (high_fee + low_fee + med_fee + high_fee)); + } + // Feerate just below tx5: tx7 and tx8 have different bump fees. + { + const auto just_below_tx5 = CFeeRate(tx5_feerate.GetFeePerK() - 5); + node::MiniMiner mini_miner(pool, all_unspent_outpoints); + BOOST_CHECK(mini_miner.IsReadyToCalculate()); + auto bump_fees = mini_miner.CalculateBumpFees(just_below_tx5); + BOOST_CHECK(!mini_miner.IsReadyToCalculate()); + BOOST_CHECK_EQUAL(bump_fees.size(), all_unspent_outpoints.size()); + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + const auto tx7_bumpfee = bump_fees.find(COutPoint{tx7->GetHash(), 0}); + BOOST_CHECK(tx7_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx7_bumpfee->second, just_below_tx5.GetFee(tx_vsizes[5] + tx_vsizes[6]) - (low_fee + med_fee)); + const auto tx8_bumpfee = bump_fees.find(COutPoint{tx8->GetHash(), 0}); + BOOST_CHECK(tx8_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx8_bumpfee->second, just_below_tx5.GetFee(tx_vsizes[5] + tx_vsizes[7]) - (low_fee + high_fee)); + // Total fees: if spending both tx7 and tx8, don't double-count fees. + node::MiniMiner mini_miner_tx7_tx8(pool, {COutPoint{tx7->GetHash(), 0}, COutPoint{tx8->GetHash(), 0}}); + BOOST_CHECK(mini_miner_tx7_tx8.IsReadyToCalculate()); + const auto tx7_tx8_bumpfee = mini_miner_tx7_tx8.CalculateTotalBumpFees(just_below_tx5); + BOOST_CHECK(!mini_miner_tx7_tx8.IsReadyToCalculate()); + BOOST_CHECK(tx7_tx8_bumpfee.has_value()); + BOOST_CHECK_EQUAL(tx7_tx8_bumpfee.value(), just_below_tx5.GetFee(tx_vsizes[5] + tx_vsizes[6]) - (low_fee + med_fee)); + } + // Feerate between tx7 and tx8's ancestor feerates: don't need to bump tx6 because tx8 already does. + { + const auto just_above_tx7 = CFeeRate(med_fee + 10, tx_vsizes[6]); + BOOST_CHECK(just_above_tx7 <= CFeeRate(low_fee + high_fee, tx_vsizes[5] + tx_vsizes[7])); + node::MiniMiner mini_miner(pool, all_unspent_outpoints); + BOOST_CHECK(mini_miner.IsReadyToCalculate()); + auto bump_fees = mini_miner.CalculateBumpFees(just_above_tx7); + BOOST_CHECK(!mini_miner.IsReadyToCalculate()); + BOOST_CHECK_EQUAL(bump_fees.size(), all_unspent_outpoints.size()); + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + const auto tx7_bumpfee = bump_fees.find(COutPoint{tx7->GetHash(), 0}); + BOOST_CHECK(tx7_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx7_bumpfee->second, just_above_tx7.GetFee(tx_vsizes[6]) - (med_fee)); + const auto tx8_bumpfee = bump_fees.find(COutPoint{tx8->GetHash(), 0}); + BOOST_CHECK(tx8_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx8_bumpfee->second, 0); + } +} +BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(cs_main, pool.cs); + + // Add chain of size 500 + TestMemPoolEntryHelper entry; + std::vector<uint256> chain_txids; + auto& lasttx = m_coinbase_txns[0]; + for (auto i{0}; i < 500; ++i) { + const auto tx = make_tx({COutPoint{lasttx->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(CENT).FromTx(tx)); + chain_txids.push_back(tx->GetHash()); + lasttx = tx; + } + const auto cluster_500tx = pool.GatherClusters({lasttx->GetHash()}); + CTxMemPool::setEntries cluster_500tx_set{cluster_500tx.begin(), cluster_500tx.end()}; + BOOST_CHECK_EQUAL(cluster_500tx.size(), cluster_500tx_set.size()); + const auto vec_iters_500 = pool.GetIterVec(chain_txids); + for (const auto& iter : vec_iters_500) BOOST_CHECK(cluster_500tx_set.count(iter)); + + // GatherClusters stops at 500 transactions. + const auto tx_501 = make_tx({COutPoint{lasttx->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(CENT).FromTx(tx_501)); + const auto cluster_501 = pool.GatherClusters({tx_501->GetHash()}); + BOOST_CHECK_EQUAL(cluster_501.size(), 0); + + // Zig Zag cluster: + // txp0 txp1 txp2 ... txp48 txp49 + // \ / \ / \ \ / + // txc0 txc1 txc2 ... txc48 + // Note that each transaction's ancestor size is 1 or 3, and each descendant size is 1, 2 or 3. + // However, all of these transactions are in the same cluster. + std::vector<uint256> zigzag_txids; + for (auto p{0}; p < 50; ++p) { + const auto txp = make_tx({COutPoint{GetRandHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(CENT).FromTx(txp)); + zigzag_txids.push_back(txp->GetHash()); + } + for (auto c{0}; c < 49; ++c) { + const auto txc = make_tx({COutPoint{zigzag_txids[c], 1}, COutPoint{zigzag_txids[c+1], 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(CENT).FromTx(txc)); + zigzag_txids.push_back(txc->GetHash()); + } + const auto vec_iters_zigzag = pool.GetIterVec(zigzag_txids); + // It doesn't matter which tx we calculate cluster for, everybody is in it. + const std::vector<size_t> indeces{0, 22, 72, zigzag_txids.size() - 1}; + for (const auto index : indeces) { + const auto cluster = pool.GatherClusters({zigzag_txids[index]}); + BOOST_CHECK_EQUAL(cluster.size(), zigzag_txids.size()); + CTxMemPool::setEntries clusterset{cluster.begin(), cluster.end()}; + BOOST_CHECK_EQUAL(cluster.size(), clusterset.size()); + for (const auto& iter : vec_iters_zigzag) BOOST_CHECK(clusterset.count(iter)); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 631c213627..aa577f7b27 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -135,7 +135,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) CNetAddr addr; // IPv4, INADDR_ANY - BOOST_REQUIRE(LookupHost("0.0.0.0", addr, false)); + addr = LookupHost("0.0.0.0", false).value(); BOOST_REQUIRE(!addr.IsValid()); BOOST_REQUIRE(addr.IsIPv4()); @@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "0.0.0.0"); // IPv4, INADDR_NONE - BOOST_REQUIRE(LookupHost("255.255.255.255", addr, false)); + addr = LookupHost("255.255.255.255", false).value(); BOOST_REQUIRE(!addr.IsValid()); BOOST_REQUIRE(addr.IsIPv4()); @@ -153,7 +153,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "255.255.255.255"); // IPv4, casual - BOOST_REQUIRE(LookupHost("12.34.56.78", addr, false)); + addr = LookupHost("12.34.56.78", false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv4()); @@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "12.34.56.78"); // IPv6, in6addr_any - BOOST_REQUIRE(LookupHost("::", addr, false)); + addr = LookupHost("::", false).value(); BOOST_REQUIRE(!addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); @@ -171,7 +171,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "::"); // IPv6, casual - BOOST_REQUIRE(LookupHost("1122:3344:5566:7788:9900:aabb:ccdd:eeff", addr, false)); + addr = LookupHost("1122:3344:5566:7788:9900:aabb:ccdd:eeff", false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); @@ -186,14 +186,14 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) // id of "32", return the address as "fe80::1%32". const std::string link_local{"fe80::1"}; const std::string scoped_addr{link_local + "%32"}; - BOOST_REQUIRE(LookupHost(scoped_addr, addr, false)); + addr = LookupHost(scoped_addr, false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); BOOST_CHECK(!addr.IsBindAny()); BOOST_CHECK_EQUAL(addr.ToStringAddr(), scoped_addr); // Test that the delimiter "%" and default zone id of 0 can be omitted for the default scope. - BOOST_REQUIRE(LookupHost(link_local + "%0", addr, false)); + addr = LookupHost(link_local + "%0", false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); BOOST_CHECK(!addr.IsBindAny()); @@ -318,10 +318,9 @@ BOOST_AUTO_TEST_CASE(cnetaddr_tostring_canonical_ipv6) {"2001:db8:aaaa:bbbb:cccc:dddd:eeee:AaAa", "2001:db8:aaaa:bbbb:cccc:dddd:eeee:aaaa"}, }; for (const auto& [input_address, expected_canonical_representation_output] : canonical_representations_ipv6) { - CNetAddr net_addr; - BOOST_REQUIRE(LookupHost(input_address, net_addr, false)); - BOOST_REQUIRE(net_addr.IsIPv6()); - BOOST_CHECK_EQUAL(net_addr.ToStringAddr(), expected_canonical_representation_output); + const std::optional<CNetAddr> net_addr{LookupHost(input_address, false)}; + BOOST_REQUIRE(net_addr.value().IsIPv6()); + BOOST_CHECK_EQUAL(net_addr.value().ToStringAddr(), expected_canonical_representation_output); } } @@ -334,12 +333,12 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v1) BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000000000000000"); s.clear(); - BOOST_REQUIRE(LookupHost("1.2.3.4", addr, false)); + addr = LookupHost("1.2.3.4", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000ffff01020304"); s.clear(); - BOOST_REQUIRE(LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", addr, false)); + addr = LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "1a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); s.clear(); @@ -370,12 +369,12 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v2) BOOST_CHECK_EQUAL(HexStr(s), "021000000000000000000000000000000000"); s.clear(); - BOOST_REQUIRE(LookupHost("1.2.3.4", addr, false)); + addr = LookupHost("1.2.3.4", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "010401020304"); s.clear(); - BOOST_REQUIRE(LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", addr, false)); + addr = LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "02101a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); s.clear(); diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 7e91819ddc..05953bfd10 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -24,9 +24,7 @@ BOOST_FIXTURE_TEST_SUITE(netbase_tests, BasicTestingSetup) static CNetAddr ResolveIP(const std::string& ip) { - CNetAddr addr; - LookupHost(ip, addr, false); - return addr; + return LookupHost(ip, false).value_or(CNetAddr{}); } static CSubNet ResolveSubNet(const std::string& subnet) @@ -477,11 +475,10 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) { - CNetAddr addr; - BOOST_CHECK(LookupHost("127.0.0.1"s, addr, false)); - BOOST_CHECK(!LookupHost("127.0.0.1\0"s, addr, false)); - BOOST_CHECK(!LookupHost("127.0.0.1\0example.com"s, addr, false)); - BOOST_CHECK(!LookupHost("127.0.0.1\0example.com\0"s, addr, false)); + BOOST_CHECK(LookupHost("127.0.0.1"s, false).has_value()); + BOOST_CHECK(!LookupHost("127.0.0.1\0"s, false).has_value()); + BOOST_CHECK(!LookupHost("127.0.0.1\0example.com"s, false).has_value()); + BOOST_CHECK(!LookupHost("127.0.0.1\0example.com\0"s, false).has_value()); CSubNet ret; BOOST_CHECK(LookupSubNet("1.2.3.0/24"s, ret)); BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0"s, ret)); diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index addc925bab..5bd14f22c6 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -7,6 +7,7 @@ #include <pow.h> #include <test/util/random.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <boost/test/unit_test.hpp> @@ -15,7 +16,7 @@ BOOST_FIXTURE_TEST_SUITE(pow_tests, BasicTestingSetup) /* Test calculation of next difficulty target with no constraints applying */ BOOST_AUTO_TEST_CASE(get_next_work) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); int64_t nLastRetargetTime = 1261130161; // Block #30240 CBlockIndex pindexLast; pindexLast.nHeight = 32255; @@ -34,7 +35,7 @@ BOOST_AUTO_TEST_CASE(get_next_work) /* Test the constraint on the upper bound for next work */ BOOST_AUTO_TEST_CASE(get_next_work_pow_limit) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); int64_t nLastRetargetTime = 1231006505; // Block #0 CBlockIndex pindexLast; pindexLast.nHeight = 2015; @@ -48,7 +49,7 @@ BOOST_AUTO_TEST_CASE(get_next_work_pow_limit) /* Test the constraint on the lower bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_lower_limit_actual) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); int64_t nLastRetargetTime = 1279008237; // Block #66528 CBlockIndex pindexLast; pindexLast.nHeight = 68543; @@ -65,7 +66,7 @@ BOOST_AUTO_TEST_CASE(get_next_work_lower_limit_actual) /* Test the constraint on the upper bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_upper_limit_actual) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); int64_t nLastRetargetTime = 1263163443; // NOTE: Not an actual block time CBlockIndex pindexLast; pindexLast.nHeight = 46367; @@ -81,7 +82,7 @@ BOOST_AUTO_TEST_CASE(get_next_work_upper_limit_actual) BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_negative_target) { - const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); + const auto consensus = CreateChainParams(*m_node.args, ChainType::MAIN)->GetConsensus(); uint256 hash; unsigned int nBits; nBits = UintToArith256(consensus.powLimit).GetCompact(true); @@ -91,7 +92,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_negative_target) BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_overflow_target) { - const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); + const auto consensus = CreateChainParams(*m_node.args, ChainType::MAIN)->GetConsensus(); uint256 hash; unsigned int nBits{~0x00800000U}; hash.SetHex("0x1"); @@ -100,7 +101,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_overflow_target) BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_too_easy_target) { - const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); + const auto consensus = CreateChainParams(*m_node.args, ChainType::MAIN)->GetConsensus(); uint256 hash; unsigned int nBits; arith_uint256 nBits_arith = UintToArith256(consensus.powLimit); @@ -112,7 +113,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_too_easy_target) BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_biger_hash_than_target) { - const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); + const auto consensus = CreateChainParams(*m_node.args, ChainType::MAIN)->GetConsensus(); uint256 hash; unsigned int nBits; arith_uint256 hash_arith = UintToArith256(consensus.powLimit); @@ -124,7 +125,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_biger_hash_than_target) BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_zero_target) { - const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); + const auto consensus = CreateChainParams(*m_node.args, ChainType::MAIN)->GetConsensus(); uint256 hash; unsigned int nBits; arith_uint256 hash_arith{0}; @@ -135,7 +136,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_zero_target) BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); std::vector<CBlockIndex> blocks(10000); for (int i = 0; i < 10000; i++) { blocks[i].pprev = i ? &blocks[i - 1] : nullptr; @@ -155,9 +156,9 @@ BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) } } -void sanity_check_chainparams(const ArgsManager& args, std::string chainName) +void sanity_check_chainparams(const ArgsManager& args, ChainType chain_type) { - const auto chainParams = CreateChainParams(args, chainName); + const auto chainParams = CreateChainParams(args, chain_type); const auto consensus = chainParams->GetConsensus(); // hash genesis is correct @@ -184,22 +185,22 @@ void sanity_check_chainparams(const ArgsManager& args, std::string chainName) BOOST_AUTO_TEST_CASE(ChainParams_MAIN_sanity) { - sanity_check_chainparams(*m_node.args, CBaseChainParams::MAIN); + sanity_check_chainparams(*m_node.args, ChainType::MAIN); } BOOST_AUTO_TEST_CASE(ChainParams_REGTEST_sanity) { - sanity_check_chainparams(*m_node.args, CBaseChainParams::REGTEST); + sanity_check_chainparams(*m_node.args, ChainType::REGTEST); } BOOST_AUTO_TEST_CASE(ChainParams_TESTNET_sanity) { - sanity_check_chainparams(*m_node.args, CBaseChainParams::TESTNET); + sanity_check_chainparams(*m_node.args, ChainType::TESTNET); } BOOST_AUTO_TEST_CASE(ChainParams_SIGNET_sanity) { - sanity_check_chainparams(*m_node.args, CBaseChainParams::SIGNET); + sanity_check_chainparams(*m_node.args, ChainType::SIGNET); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp index 0ec253747b..10205cd641 100644 --- a/src/test/rbf_tests.cpp +++ b/src/test/rbf_tests.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2021-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <policy/rbf.h> #include <random.h> #include <test/util/txmempool.h> #include <txmempool.h> -#include <util/system.h> #include <util/time.h> #include <test/util/setup_common.h> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 791c9ddf31..2f783a4b95 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -42,11 +42,11 @@ private: class RPCTestingSetup : public TestingSetup { public: - UniValue TransformParams(const UniValue& params, std::vector<std::string> arg_names) const; + UniValue TransformParams(const UniValue& params, std::vector<std::pair<std::string, bool>> arg_names) const; UniValue CallRPC(std::string args); }; -UniValue RPCTestingSetup::TransformParams(const UniValue& params, std::vector<std::string> arg_names) const +UniValue RPCTestingSetup::TransformParams(const UniValue& params, std::vector<std::pair<std::string, bool>> arg_names) const { UniValue transformed_params; CRPCTable table; @@ -75,7 +75,7 @@ UniValue RPCTestingSetup::CallRPC(std::string args) return result; } catch (const UniValue& objError) { - throw std::runtime_error(find_value(objError, "message").get_str()); + throw std::runtime_error(objError.find_value("message").get_str()); } } @@ -84,7 +84,7 @@ BOOST_FIXTURE_TEST_SUITE(rpc_tests, RPCTestingSetup) BOOST_AUTO_TEST_CASE(rpc_namedparams) { - const std::vector<std::string> arg_names{"arg1", "arg2", "arg3", "arg4", "arg5"}; + const std::vector<std::pair<std::string, bool>> arg_names{{"arg1", false}, {"arg2", false}, {"arg3", false}, {"arg4", false}, {"arg5", false}}; // Make sure named arguments are transformed into positional arguments in correct places separated by nulls BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg2": 2, "arg4": 4})"), arg_names).write(), "[null,2,null,4]"); @@ -109,6 +109,28 @@ BOOST_AUTO_TEST_CASE(rpc_namedparams) BOOST_CHECK_EQUAL(TransformParams(JSON(R"([1,2,3,4,5,6,7,8,9,10])"), arg_names).write(), "[1,2,3,4,5,6,7,8,9,10]"); } +BOOST_AUTO_TEST_CASE(rpc_namedonlyparams) +{ + const std::vector<std::pair<std::string, bool>> arg_names{{"arg1", false}, {"arg2", false}, {"opt1", true}, {"opt2", true}, {"options", false}}; + + // Make sure optional parameters are really optional. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg1": 1, "arg2": 2})"), arg_names).write(), "[1,2]"); + + // Make sure named-only parameters are passed as options. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg1": 1, "arg2": 2, "opt1": 10, "opt2": 20})"), arg_names).write(), R"([1,2,{"opt1":10,"opt2":20}])"); + + // Make sure options can be passed directly. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg1": 1, "arg2": 2, "options":{"opt1": 10, "opt2": 20}})"), arg_names).write(), R"([1,2,{"opt1":10,"opt2":20}])"); + + // Make sure options and named parameters conflict. + BOOST_CHECK_EXCEPTION(TransformParams(JSON(R"({"arg1": 1, "arg2": 2, "opt1": 10, "options":{"opt1": 10}})"), arg_names), UniValue, + HasJSON(R"({"code":-8,"message":"Parameter options conflicts with parameter opt1"})")); + + // Make sure options object specified through args array conflicts. + BOOST_CHECK_EXCEPTION(TransformParams(JSON(R"({"args": [1, 2, {"opt1": 10}], "opt2": 20})"), arg_names), UniValue, + HasJSON(R"({"code":-8,"message":"Parameter options specified twice both as positional and named argument"})")); +} + BOOST_AUTO_TEST_CASE(rpc_rawparams) { // Test raw transaction API argument handling @@ -130,9 +152,9 @@ BOOST_AUTO_TEST_CASE(rpc_rawparams) BOOST_CHECK_THROW(CallRPC("decoderawtransaction DEADBEEF"), std::runtime_error); std::string rawtx = "0100000001a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff0100e1f505000000001976a9140389035a9225b3839e2bbf32d826a1e222031fd888ac00000000"; BOOST_CHECK_NO_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx)); - BOOST_CHECK_EQUAL(find_value(r.get_obj(), "size").getInt<int>(), 193); - BOOST_CHECK_EQUAL(find_value(r.get_obj(), "version").getInt<int>(), 1); - BOOST_CHECK_EQUAL(find_value(r.get_obj(), "locktime").getInt<int>(), 0); + BOOST_CHECK_EQUAL(r.get_obj().find_value("size").getInt<int>(), 193); + BOOST_CHECK_EQUAL(r.get_obj().find_value("version").getInt<int>(), 1); + BOOST_CHECK_EQUAL(r.get_obj().find_value("locktime").getInt<int>(), 0); BOOST_CHECK_THROW(CallRPC(std::string("decoderawtransaction ")+rawtx+" extra"), std::runtime_error); BOOST_CHECK_NO_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx+" false")); BOOST_CHECK_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx+" false extra"), std::runtime_error); @@ -149,20 +171,20 @@ BOOST_AUTO_TEST_CASE(rpc_togglenetwork) UniValue r; r = CallRPC("getnetworkinfo"); - bool netState = find_value(r.get_obj(), "networkactive").get_bool(); + bool netState = r.get_obj().find_value("networkactive").get_bool(); BOOST_CHECK_EQUAL(netState, true); BOOST_CHECK_NO_THROW(CallRPC("setnetworkactive false")); r = CallRPC("getnetworkinfo"); - int numConnection = find_value(r.get_obj(), "connections").getInt<int>(); + int numConnection = r.get_obj().find_value("connections").getInt<int>(); BOOST_CHECK_EQUAL(numConnection, 0); - netState = find_value(r.get_obj(), "networkactive").get_bool(); + netState = r.get_obj().find_value("networkactive").get_bool(); BOOST_CHECK_EQUAL(netState, false); BOOST_CHECK_NO_THROW(CallRPC("setnetworkactive true")); r = CallRPC("getnetworkinfo"); - netState = find_value(r.get_obj(), "networkactive").get_bool(); + netState = r.get_obj().find_value("networkactive").get_bool(); BOOST_CHECK_EQUAL(netState, true); } @@ -180,9 +202,9 @@ BOOST_AUTO_TEST_CASE(rpc_rawsign) std::string privkey1 = "\"KzsXybp9jX64P5ekX1KUxRQ79Jht9uzW7LorgwE65i5rWACL6LQe\""; std::string privkey2 = "\"Kyhdf5LuKTRx4ge69ybABsiUAWjVRK4XGxAKk2FQLp2HjGMy87Z4\""; r = CallRPC(std::string("signrawtransactionwithkey ")+notsigned+" [] "+prevout); - BOOST_CHECK(find_value(r.get_obj(), "complete").get_bool() == false); + BOOST_CHECK(r.get_obj().find_value("complete").get_bool() == false); r = CallRPC(std::string("signrawtransactionwithkey ")+notsigned+" ["+privkey1+","+privkey2+"] "+prevout); - BOOST_CHECK(find_value(r.get_obj(), "complete").get_bool() == true); + BOOST_CHECK(r.get_obj().find_value("complete").get_bool() == true); } BOOST_AUTO_TEST_CASE(rpc_createraw_op_return) @@ -278,6 +300,7 @@ BOOST_AUTO_TEST_CASE(rpc_parse_monetary_values) BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.00000001000000")), 1LL); //should pass, cut trailing 0 BOOST_CHECK_THROW(AmountFromValue(ValueFromString("19e-9")), UniValue); //should fail BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.19e-6")), 19); //should pass, leading 0 is present + BOOST_CHECK_EXCEPTION(AmountFromValue(".19e-6"), UniValue, HasJSON(R"({"code":-3,"message":"Invalid amount"})")); //should fail, no leading 0 BOOST_CHECK_THROW(AmountFromValue(ValueFromString("92233720368.54775808")), UniValue); //overflow error BOOST_CHECK_THROW(AmountFromValue(ValueFromString("1e+11")), UniValue); //overflow error @@ -285,36 +308,6 @@ BOOST_AUTO_TEST_CASE(rpc_parse_monetary_values) BOOST_CHECK_THROW(AmountFromValue(ValueFromString("93e+9")), UniValue); //overflow error } -BOOST_AUTO_TEST_CASE(json_parse_errors) -{ - // Valid - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("1.0").get_real(), 1.0); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("true").get_bool(), true); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("[false]")[0].get_bool(), false); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("{\"a\": true}")["a"].get_bool(), true); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("{\"1\": \"true\"}")["1"].get_str(), "true"); - // Valid, with leading or trailing whitespace - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue(" 1.0").get_real(), 1.0); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("1.0 ").get_real(), 1.0); - - BOOST_CHECK_THROW(AmountFromValue(ParseNonRFCJSONValue(".19e-6")), std::runtime_error); //should fail, missing leading 0, therefore invalid JSON - BOOST_CHECK_EQUAL(AmountFromValue(ParseNonRFCJSONValue("0.00000000000000000000000000000000000001e+30 ")), 1); - // Invalid, initial garbage - BOOST_CHECK_THROW(ParseNonRFCJSONValue("[1.0"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("a1.0"), std::runtime_error); - // Invalid, trailing garbage - BOOST_CHECK_THROW(ParseNonRFCJSONValue("1.0sds"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("1.0]"), std::runtime_error); - // Invalid, keys have to be names - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{1: \"true\"}"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{true: 1}"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{[1]: 1}"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{{\"a\": \"a\"}: 1}"), std::runtime_error); - // BTC addresses should fail parsing - BOOST_CHECK_THROW(ParseNonRFCJSONValue("175tWpb8K1S7NmH4Zx6rewF9WQrcZv245W"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("3J98t1WpEZ73CNmQviecrnyiWrnqRhWNL"), std::runtime_error); -} - BOOST_AUTO_TEST_CASE(rpc_ban) { BOOST_CHECK_NO_THROW(CallRPC(std::string("clearbanned"))); @@ -325,7 +318,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); UniValue ar = r.get_array(); UniValue o1 = ar[0].get_obj(); - UniValue adr = find_value(o1, "address"); + UniValue adr = o1.find_value("address"); BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/32"); BOOST_CHECK_NO_THROW(CallRPC(std::string("setban 127.0.0.0 remove"))); BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); @@ -336,8 +329,8 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); - adr = find_value(o1, "address"); - int64_t banned_until{find_value(o1, "banned_until").getInt<int64_t>()}; + adr = o1.find_value("address"); + int64_t banned_until{o1.find_value("banned_until").getInt<int64_t>()}; BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/24"); BOOST_CHECK_EQUAL(banned_until, 9907731200); // absolute time check @@ -351,11 +344,11 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); - adr = find_value(o1, "address"); - banned_until = find_value(o1, "banned_until").getInt<int64_t>(); - const int64_t ban_created{find_value(o1, "ban_created").getInt<int64_t>()}; - const int64_t ban_duration{find_value(o1, "ban_duration").getInt<int64_t>()}; - const int64_t time_remaining{find_value(o1, "time_remaining").getInt<int64_t>()}; + adr = o1.find_value("address"); + banned_until = o1.find_value("banned_until").getInt<int64_t>(); + const int64_t ban_created{o1.find_value("ban_created").getInt<int64_t>()}; + const int64_t ban_duration{o1.find_value("ban_duration").getInt<int64_t>()}; + const int64_t time_remaining{o1.find_value("time_remaining").getInt<int64_t>()}; BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/24"); BOOST_CHECK_EQUAL(banned_until, time_remaining_expected + now.count()); BOOST_CHECK_EQUAL(ban_duration, banned_until - ban_created); @@ -385,7 +378,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); - adr = find_value(o1, "address"); + adr = o1.find_value("address"); BOOST_CHECK_EQUAL(adr.get_str(), "fe80::202:b3ff:fe1e:8329/128"); BOOST_CHECK_NO_THROW(CallRPC(std::string("clearbanned"))); @@ -393,7 +386,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); - adr = find_value(o1, "address"); + adr = o1.find_value("address"); BOOST_CHECK_EQUAL(adr.get_str(), "2001:db8::/30"); BOOST_CHECK_NO_THROW(CallRPC(std::string("clearbanned"))); @@ -401,7 +394,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); - adr = find_value(o1, "address"); + adr = o1.find_value("address"); BOOST_CHECK_EQUAL(adr.get_str(), "2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/128"); } @@ -506,6 +499,53 @@ BOOST_AUTO_TEST_CASE(rpc_getblockstats_calculate_percentiles_by_weight) } } +// Make sure errors are triggered appropriately if parameters have the same names. +BOOST_AUTO_TEST_CASE(check_dup_param_names) +{ + enum ParamType { POSITIONAL, NAMED, NAMED_ONLY }; + auto make_rpc = [](std::vector<std::tuple<std::string, ParamType>> param_names) { + std::vector<RPCArg> params; + std::vector<RPCArg> options; + auto push_options = [&] { if (!options.empty()) params.emplace_back(RPCArg{strprintf("options%i", params.size()), RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", std::move(options)}); }; + for (auto& [param_name, param_type] : param_names) { + if (param_type == POSITIONAL) { + push_options(); + params.emplace_back(std::move(param_name), RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "description"); + } else { + options.emplace_back(std::move(param_name), RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "description", RPCArgOptions{.also_positional = param_type == NAMED}); + } + } + push_options(); + return RPCHelpMan{"method_name", "description", params, RPCResults{}, RPCExamples{""}}; + }; + + // No errors if parameter names are unique. + make_rpc({{"p1", POSITIONAL}, {"p2", POSITIONAL}}); + make_rpc({{"p1", POSITIONAL}, {"p2", NAMED}}); + make_rpc({{"p1", POSITIONAL}, {"p2", NAMED_ONLY}}); + make_rpc({{"p1", NAMED}, {"p2", POSITIONAL}}); + make_rpc({{"p1", NAMED}, {"p2", NAMED}}); + make_rpc({{"p1", NAMED}, {"p2", NAMED_ONLY}}); + make_rpc({{"p1", NAMED_ONLY}, {"p2", POSITIONAL}}); + make_rpc({{"p1", NAMED_ONLY}, {"p2", NAMED}}); + make_rpc({{"p1", NAMED_ONLY}, {"p2", NAMED_ONLY}}); + + // Error if parameters names are duplicates, unless one parameter is + // positional and the other is named and .also_positional is true. + BOOST_CHECK_THROW(make_rpc({{"p1", POSITIONAL}, {"p1", POSITIONAL}}), NonFatalCheckError); + make_rpc({{"p1", POSITIONAL}, {"p1", NAMED}}); + BOOST_CHECK_THROW(make_rpc({{"p1", POSITIONAL}, {"p1", NAMED_ONLY}}), NonFatalCheckError); + make_rpc({{"p1", NAMED}, {"p1", POSITIONAL}}); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED}, {"p1", NAMED}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED}, {"p1", NAMED_ONLY}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED_ONLY}, {"p1", POSITIONAL}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED_ONLY}, {"p1", NAMED}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED_ONLY}, {"p1", NAMED_ONLY}}), NonFatalCheckError); + + // Make sure duplicate aliases are detected too. + BOOST_CHECK_THROW(make_rpc({{"p1", POSITIONAL}, {"p2|p1", NAMED_ONLY}}), NonFatalCheckError); +} + BOOST_AUTO_TEST_CASE(help_example) { // test different argument types diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index d8898743b0..c89f2c1f5b 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -5,6 +5,7 @@ #include <test/data/script_tests.json.h> #include <test/data/bip341_wallet_vectors.json.h> +#include <common/system.h> #include <core_io.h> #include <key.h> #include <rpc/util.h> @@ -20,7 +21,6 @@ #include <test/util/transaction_utils.h> #include <util/fs.h> #include <util/strencodings.h> -#include <util/system.h> #if defined(HAVE_CONSENSUS_LIB) #include <script/bitcoinconsensus.h> diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 604eb48df6..c24921bf9b 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -2,16 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/settings.h> +#include <common/settings.h> #include <test/util/setup_common.h> #include <test/util/str.h> -#include <util/fs.h> - #include <boost/test/unit_test.hpp> #include <common/args.h> #include <univalue.h> +#include <util/chaintype.h> +#include <util/fs.h> #include <util/strencodings.h> #include <util/string.h> @@ -21,20 +21,20 @@ #include <system_error> #include <vector> -inline bool operator==(const util::SettingsValue& a, const util::SettingsValue& b) +inline bool operator==(const common::SettingsValue& a, const common::SettingsValue& b) { return a.write() == b.write(); } -inline std::ostream& operator<<(std::ostream& os, const util::SettingsValue& value) +inline std::ostream& operator<<(std::ostream& os, const common::SettingsValue& value) { os << value.write(); return os; } -inline std::ostream& operator<<(std::ostream& os, const std::pair<std::string, util::SettingsValue>& kv) +inline std::ostream& operator<<(std::ostream& os, const std::pair<std::string, common::SettingsValue>& kv) { - util::SettingsValue out(util::SettingsValue::VOBJ); + common::SettingsValue out(common::SettingsValue::VOBJ); out.__pushKV(kv.first, kv.second); os << out.write(); return os; @@ -60,7 +60,7 @@ BOOST_AUTO_TEST_CASE(ReadWrite) "null": null })"); - std::map<std::string, util::SettingsValue> expected{ + std::map<std::string, common::SettingsValue> expected{ {"string", "string"}, {"num", 5}, {"bool", true}, @@ -68,15 +68,15 @@ BOOST_AUTO_TEST_CASE(ReadWrite) }; // Check file read. - std::map<std::string, util::SettingsValue> values; + std::map<std::string, common::SettingsValue> values; std::vector<std::string> errors; - BOOST_CHECK(util::ReadSettings(path, values, errors)); + BOOST_CHECK(common::ReadSettings(path, values, errors)); BOOST_CHECK_EQUAL_COLLECTIONS(values.begin(), values.end(), expected.begin(), expected.end()); BOOST_CHECK(errors.empty()); // Check no errors if file doesn't exist. fs::remove(path); - BOOST_CHECK(util::ReadSettings(path, values, errors)); + BOOST_CHECK(common::ReadSettings(path, values, errors)); BOOST_CHECK(values.empty()); BOOST_CHECK(errors.empty()); @@ -85,29 +85,29 @@ BOOST_AUTO_TEST_CASE(ReadWrite) "dupe": "string", "dupe": "dupe" })"); - BOOST_CHECK(!util::ReadSettings(path, values, errors)); + BOOST_CHECK(!common::ReadSettings(path, values, errors)); std::vector<std::string> dup_keys = {strprintf("Found duplicate key dupe in settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), dup_keys.begin(), dup_keys.end()); BOOST_CHECK(values.empty()); // Check non-kv json files not allowed WriteText(path, R"("non-kv")"); - BOOST_CHECK(!util::ReadSettings(path, values, errors)); + BOOST_CHECK(!common::ReadSettings(path, values, errors)); std::vector<std::string> non_kv = {strprintf("Found non-object value \"non-kv\" in settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), non_kv.begin(), non_kv.end()); // Check invalid json not allowed WriteText(path, R"(invalid json)"); - BOOST_CHECK(!util::ReadSettings(path, values, errors)); + BOOST_CHECK(!common::ReadSettings(path, values, errors)); std::vector<std::string> fail_parse = {strprintf("Unable to parse settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), fail_parse.begin(), fail_parse.end()); } //! Check settings struct contents against expected json strings. -static void CheckValues(const util::Settings& settings, const std::string& single_val, const std::string& list_val) +static void CheckValues(const common::Settings& settings, const std::string& single_val, const std::string& list_val) { - util::SettingsValue single_value = GetSetting(settings, "section", "name", false, false, false); - util::SettingsValue list_value(util::SettingsValue::VARR); + common::SettingsValue single_value = GetSetting(settings, "section", "name", false, false, false); + common::SettingsValue list_value(common::SettingsValue::VARR); for (const auto& item : GetSettingsList(settings, "section", "name", false)) { list_value.push_back(item); } @@ -118,7 +118,7 @@ static void CheckValues(const util::Settings& settings, const std::string& singl // Simple settings merge test case. BOOST_AUTO_TEST_CASE(Simple) { - util::Settings settings; + common::Settings settings; settings.command_line_options["name"].push_back("val1"); settings.command_line_options["name"].push_back("val2"); settings.ro_config["section"]["name"].push_back(2); @@ -126,7 +126,7 @@ BOOST_AUTO_TEST_CASE(Simple) // The last given arg takes precedence when specified via commandline. CheckValues(settings, R"("val2")", R"(["val1","val2",2])"); - util::Settings settings2; + common::Settings settings2; settings2.ro_config["section"]["name"].push_back("val2"); settings2.ro_config["section"]["name"].push_back("val3"); @@ -140,7 +140,7 @@ BOOST_AUTO_TEST_CASE(Simple) // its default value. BOOST_AUTO_TEST_CASE(NullOverride) { - util::Settings settings; + common::Settings settings; settings.command_line_options["name"].push_back("value"); BOOST_CHECK_EQUAL(R"("value")", GetSetting(settings, "section", "name", false, false, false).write().c_str()); settings.forced_settings["name"] = {}; @@ -190,16 +190,16 @@ BOOST_FIXTURE_TEST_CASE(Merge, MergeTestingSetup) if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); } - const std::string& network = CBaseChainParams::MAIN; + const std::string& network = ChainTypeToString(ChainType::MAIN); ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions, bool force_set, bool ignore_default_section_config) { std::string desc; int value_suffix = 0; - util::Settings settings; + common::Settings settings; const std::string& name = ignore_default_section_config ? "wallet" : "server"; auto push_values = [&](Action action, const char* value_prefix, const std::string& name_prefix, - std::vector<util::SettingsValue>& dest) { + std::vector<common::SettingsValue>& dest) { if (action == SET || action == SECTION_SET) { for (int i = 0; i < 2; ++i) { dest.push_back(value_prefix + ToString(++value_suffix)); @@ -225,7 +225,7 @@ BOOST_FIXTURE_TEST_CASE(Merge, MergeTestingSetup) } desc += " || "; - desc += GetSetting(settings, network, name, ignore_default_section_config, /*ignore_nonpersistent=*/false, /*get_chain_name=*/false).write(); + desc += GetSetting(settings, network, name, ignore_default_section_config, /*ignore_nonpersistent=*/false, /*get_chain_type=*/false).write(); desc += " |"; for (const auto& s : GetSettingsList(settings, network, name, ignore_default_section_config)) { desc += " "; diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index e2d11afa6a..68ef719c71 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <consensus/tx_check.h> #include <consensus/validation.h> #include <hash.h> @@ -14,7 +15,6 @@ #include <test/util/random.h> #include <test/util/setup_common.h> #include <util/strencodings.h> -#include <util/system.h> #include <version.h> #include <iostream> diff --git a/src/test/sock_tests.cpp b/src/test/sock_tests.cpp index 9e6f73745e..26ee724bf8 100644 --- a/src/test/sock_tests.cpp +++ b/src/test/sock_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <compat/compat.h> #include <test/util/setup_common.h> #include <util/sock.h> -#include <util/system.h> #include <util/threadinterrupt.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index a0392570f1..55e4f200b1 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -463,7 +463,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) size_t find = currentPos + InsecureRandRange(8); if (find >= fileSize) find = fileSize - 1; - bf.FindByte(uint8_t(find)); + bf.FindByte(std::byte(find)); // The value at each offset is the offset. BOOST_CHECK_EQUAL(bf.GetPos(), find); currentPos = find; diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index c0a2566959..7ce350b84b 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -52,7 +52,7 @@ BOOST_AUTO_TEST_CASE(run_command) const UniValue result = RunCommandParseJSON("echo \"{\"success\": true}\""); #endif BOOST_CHECK(result.isObject()); - const UniValue& success = find_value(result, "success"); + const UniValue& success = result.find_value("success"); BOOST_CHECK(!success.isNull()); BOOST_CHECK_EQUAL(success.get_bool(), true); } @@ -106,7 +106,7 @@ BOOST_AUTO_TEST_CASE(run_command) { const UniValue result = RunCommandParseJSON("cat", "{\"success\": true}"); BOOST_CHECK(result.isObject()); - const UniValue& success = find_value(result, "success"); + const UniValue& success = result.find_value("success"); BOOST_CHECK(!success.isNull()); BOOST_CHECK_EQUAL(success.get_bool(), true); } diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index b9bfa65c0a..b666517ae2 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -32,10 +32,10 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) BOOST_REQUIRE(txindex.Start()); // Allow tx index to catch up with the block index. - constexpr int64_t timeout_ms = 10 * 1000; - int64_t time_start = GetTimeMillis(); + constexpr auto timeout{10s}; + const auto time_start{SteadyClock::now()}; while (!txindex.BlockUntilSyncedToCurrentChain()) { - BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis()); + BOOST_REQUIRE(time_start + timeout > SteadyClock::now()); UninterruptibleSleep(std::chrono::milliseconds{100}); } diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 8cdea3890e..8f8628169f 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -9,13 +9,14 @@ #include <script/standard.h> #include <test/util/setup_common.h> #include <txmempool.h> +#include <util/chaintype.h> #include <validation.h> #include <boost/test/unit_test.hpp> struct Dersig100Setup : public TestChain100Setup { Dersig100Setup() - : TestChain100Setup{CBaseChainParams::REGTEST, {"-testactivationheight=dersig@102"}} {} + : TestChain100Setup{ChainType::REGTEST, {"-testactivationheight=dersig@102"}} {} }; bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, diff --git a/src/test/util/blockfilter.cpp b/src/test/util/blockfilter.cpp index ec703c6a7b..a806844e34 100644 --- a/src/test/util/blockfilter.cpp +++ b/src/test/util/blockfilter.cpp @@ -8,20 +8,19 @@ #include <node/blockstorage.h> #include <validation.h> -using node::ReadBlockFromDisk; -using node::UndoReadFromDisk; +using node::BlockManager; -bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter) +bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex& block_index, BlockFilter& filter, const BlockManager& blockman) { LOCK(::cs_main); CBlock block; - if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { + if (!blockman.ReadBlockFromDisk(block, block_index.GetBlockPos())) { return false; } CBlockUndo block_undo; - if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) { + if (block_index.nHeight > 0 && !blockman.UndoReadFromDisk(block_undo, block_index)) { return false; } diff --git a/src/test/util/blockfilter.h b/src/test/util/blockfilter.h index 79d11dcad8..789ce5d3aa 100644 --- a/src/test/util/blockfilter.h +++ b/src/test/util/blockfilter.h @@ -6,8 +6,12 @@ #define BITCOIN_TEST_UTIL_BLOCKFILTER_H #include <blockfilter.h> + class CBlockIndex; +namespace node { +class BlockManager; +} -bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter); +bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex& block_index, BlockFilter& filter, const node::BlockManager& blockman); #endif // BITCOIN_TEST_UTIL_BLOCKFILTER_H diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index 0df1db84c4..51f4b89512 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -6,19 +6,22 @@ #include <chainparams.h> #include <consensus/merkle.h> +#include <consensus/validation.h> #include <key_io.h> #include <node/context.h> #include <pow.h> +#include <primitives/transaction.h> #include <script/standard.h> #include <test/util/script.h> #include <util/check.h> #include <validation.h> +#include <validationinterface.h> #include <versionbits.h> using node::BlockAssembler; using node::NodeContext; -CTxIn generatetoaddress(const NodeContext& node, const std::string& address) +COutPoint generatetoaddress(const NodeContext& node, const std::string& address) { const auto dest = DecodeDestination(address); assert(IsValidDestination(dest)); @@ -58,19 +61,52 @@ std::vector<std::shared_ptr<CBlock>> CreateBlockChain(size_t total_height, const return ret; } -CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) +COutPoint MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) { auto block = PrepareBlock(node, coinbase_scriptPubKey); + auto valid = MineBlock(node, block); + assert(!valid.IsNull()); + return valid; +} + +struct BlockValidationStateCatcher : public CValidationInterface { + const uint256 m_hash; + std::optional<BlockValidationState> m_state; + BlockValidationStateCatcher(const uint256& hash) + : m_hash{hash}, + m_state{} {} + +protected: + void BlockChecked(const CBlock& block, const BlockValidationState& state) override + { + if (block.GetHash() != m_hash) return; + m_state = state; + } +}; + +COutPoint MineBlock(const NodeContext& node, std::shared_ptr<CBlock>& block) +{ while (!CheckProofOfWork(block->GetHash(), block->nBits, Params().GetConsensus())) { ++block->nNonce; assert(block->nNonce); } - bool processed{Assert(node.chainman)->ProcessNewBlock(block, true, true, nullptr)}; - assert(processed); - - return CTxIn{block->vtx[0]->GetHash(), 0}; + auto& chainman{*Assert(node.chainman)}; + const auto old_height = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveHeight()); + bool new_block; + BlockValidationStateCatcher bvsc{block->GetHash()}; + RegisterValidationInterface(&bvsc); + const bool processed{chainman.ProcessNewBlock(block, true, true, &new_block)}; + const bool duplicate{!new_block && processed}; + assert(!duplicate); + UnregisterValidationInterface(&bvsc); + SyncWithValidationInterfaceQueue(); + const bool was_valid{bvsc.m_state && bvsc.m_state->IsValid()}; + assert(old_height + was_valid == WITH_LOCK(chainman.GetMutex(), return chainman.ActiveHeight())); + + if (was_valid) return {block->vtx[0]->GetHash(), 0}; + return {}; } std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey, diff --git a/src/test/util/mining.h b/src/test/util/mining.h index 70b1f7b3fb..3f071257f1 100644 --- a/src/test/util/mining.h +++ b/src/test/util/mining.h @@ -13,8 +13,8 @@ class CBlock; class CChainParams; +class COutPoint; class CScript; -class CTxIn; namespace node { struct NodeContext; } // namespace node @@ -23,7 +23,13 @@ struct NodeContext; std::vector<std::shared_ptr<CBlock>> CreateBlockChain(size_t total_height, const CChainParams& params); /** Returns the generated coin */ -CTxIn MineBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); +COutPoint MineBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); + +/** + * Returns the generated coin (or Null if the block was invalid). + * It is recommended to call RegenerateCommitments before mining the block to avoid merkle tree mismatches. + **/ +COutPoint MineBlock(const node::NodeContext&, std::shared_ptr<CBlock>& block); /** Prepare a block to be mined */ std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); @@ -31,6 +37,6 @@ std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext& node, const CScrip const node::BlockAssembler::Options& assembler_options); /** RPC-like helper function, returns the generated coin */ -CTxIn generatetoaddress(const node::NodeContext&, const std::string& address); +COutPoint generatetoaddress(const node::NodeContext&, const std::string& address); #endif // BITCOIN_TEST_UTIL_MINING_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index ddc1e7ab37..483404779e 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -9,6 +9,7 @@ #include <addrman.h> #include <banman.h> #include <chainparams.h> +#include <common/system.h> #include <common/url.h> #include <consensus/consensus.h> #include <consensus/params.h> @@ -23,6 +24,7 @@ #include <node/blockstorage.h> #include <node/chainstate.h> #include <node/context.h> +#include <node/kernel_notifications.h> #include <node/mempool_args.h> #include <node/miner.h> #include <node/validation_cache_args.h> @@ -42,9 +44,9 @@ #include <timedata.h> #include <txdb.h> #include <txmempool.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/thread.h> #include <util/threadnames.h> #include <util/time.h> @@ -61,7 +63,9 @@ using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; using node::BlockAssembler; +using node::BlockManager; using node::CalculateCacheSizes; +using node::KernelNotifications; using node::LoadChainstate; using node::RegenerateCommitments; using node::VerifyLoadedChainstate; @@ -98,7 +102,7 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) return os; } -BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) +BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vector<const char*>& extra_args) : m_path_root{fs::temp_directory_path() / "test_common_" PACKAGE_NAME / g_insecure_rand_ctx_temp_path.rand256().ToString()}, m_args{} { @@ -132,7 +136,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve throw std::runtime_error{error}; } } - SelectParams(chainName); + SelectParams(chainType); SeedInsecureRand(); if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN); InitLogging(*m_node.args); @@ -153,6 +157,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve noui_connect(); noui_connected = true; } + node::g_indexes_ready_to_sync = true; } BasicTestingSetup::~BasicTestingSetup() @@ -163,8 +168,8 @@ BasicTestingSetup::~BasicTestingSetup() gArgs.ClearArgs(); } -ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) - : BasicTestingSetup(chainName, extra_args) +ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vector<const char*>& extra_args) + : BasicTestingSetup(chainType, extra_args) { const CChainParams& chainparams = Params(); @@ -179,13 +184,20 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve m_cache_sizes = CalculateCacheSizes(m_args); + m_node.notifications = std::make_unique<KernelNotifications>(); + const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = m_args.GetDataDirNet(), .adjusted_time_callback = GetAdjustedTime, .check_block_index = true, + .notifications = *m_node.notifications, + }; + const BlockManager::Options blockman_opts{ + .chainparams = chainman_opts.chainparams, + .blocks_dir = m_args.GetBlocksDirPath(), }; - m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts, node::BlockManager::Options{}); + m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts, blockman_opts); m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<CBlockTreeDB>(DBParams{ .path = m_args.GetDataDirNet() / "blocks" / "index", .cache_bytes = static_cast<size_t>(m_cache_sizes.block_tree_db), @@ -211,7 +223,7 @@ ChainTestingSetup::~ChainTestingSetup() m_node.chainman.reset(); } -void TestingSetup::LoadVerifyActivateChainstate() +void ChainTestingSetup::LoadVerifyActivateChainstate() { auto& chainman{*Assert(m_node.chainman)}; node::ChainstateLoadOptions options; @@ -237,14 +249,14 @@ void TestingSetup::LoadVerifyActivateChainstate() } TestingSetup::TestingSetup( - const std::string& chainName, + const ChainType chainType, const std::vector<const char*>& extra_args, const bool coins_db_in_memory, const bool block_tree_db_in_memory) - : ChainTestingSetup(chainName, extra_args), - m_coins_db_in_memory(coins_db_in_memory), - m_block_tree_db_in_memory(block_tree_db_in_memory) + : ChainTestingSetup(chainType, extra_args) { + m_coins_db_in_memory = coins_db_in_memory; + m_block_tree_db_in_memory = block_tree_db_in_memory; // Ideally we'd move all the RPC tests to the functional testing framework // instead of unit tests, but for now we need these here. RegisterAllCoreRPCCommands(tableRPC); @@ -268,11 +280,11 @@ TestingSetup::TestingSetup( } TestChain100Setup::TestChain100Setup( - const std::string& chain_name, + const ChainType chain_type, const std::vector<const char*>& extra_args, const bool coins_db_in_memory, const bool block_tree_db_in_memory) - : TestingSetup{CBaseChainParams::REGTEST, extra_args, coins_db_in_memory, block_tree_db_in_memory} + : TestingSetup{ChainType::REGTEST, extra_args, coins_db_in_memory, block_tree_db_in_memory} { SetMockTime(1598887952); constexpr std::array<unsigned char, 32> vchKey = { diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 750f010fb0..b7429df02c 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -5,15 +5,15 @@ #ifndef BITCOIN_TEST_UTIL_SETUP_COMMON_H #define BITCOIN_TEST_UTIL_SETUP_COMMON_H -#include <chainparamsbase.h> #include <common/args.h> #include <key.h> #include <node/caches.h> -#include <node/context.h> +#include <node/context.h> // IWYU pragma: export #include <primitives/transaction.h> #include <pubkey.h> #include <random.h> #include <stdexcept> +#include <util/chaintype.h> #include <util/check.h> #include <util/fs.h> #include <util/string.h> @@ -80,7 +80,7 @@ static constexpr CAmount CENT{1000000}; struct BasicTestingSetup { node::NodeContext m_node; // keep as first member to be destructed last - explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); + explicit BasicTestingSetup(const ChainType chainType = ChainType::MAIN, const std::vector<const char*>& extra_args = {}); ~BasicTestingSetup(); const fs::path m_path_root; @@ -93,21 +93,21 @@ struct BasicTestingSetup { */ struct ChainTestingSetup : public BasicTestingSetup { node::CacheSizes m_cache_sizes{}; + bool m_coins_db_in_memory{true}; + bool m_block_tree_db_in_memory{true}; - explicit ChainTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); + explicit ChainTestingSetup(const ChainType chainType = ChainType::MAIN, const std::vector<const char*>& extra_args = {}); ~ChainTestingSetup(); + + // Supplies a chainstate, if one is needed + void LoadVerifyActivateChainstate(); }; /** Testing setup that configures a complete environment. */ struct TestingSetup : public ChainTestingSetup { - bool m_coins_db_in_memory{true}; - bool m_block_tree_db_in_memory{true}; - - void LoadVerifyActivateChainstate(); - explicit TestingSetup( - const std::string& chainName = CBaseChainParams::MAIN, + const ChainType chainType = ChainType::MAIN, const std::vector<const char*>& extra_args = {}, const bool coins_db_in_memory = true, const bool block_tree_db_in_memory = true); @@ -116,7 +116,7 @@ struct TestingSetup : public ChainTestingSetup { /** Identical to TestingSetup, but chain set to regtest */ struct RegTestingSetup : public TestingSetup { RegTestingSetup() - : TestingSetup{CBaseChainParams::REGTEST} {} + : TestingSetup{ChainType::REGTEST} {} }; class CBlock; @@ -128,7 +128,7 @@ class CScript; */ struct TestChain100Setup : public TestingSetup { TestChain100Setup( - const std::string& chain_name = CBaseChainParams::REGTEST, + const ChainType chain_type = ChainType::REGTEST, const std::vector<const char*>& extra_args = {}, const bool coins_db_in_memory = true, const bool block_tree_db_in_memory = true); @@ -206,7 +206,7 @@ struct TestChain100Setup : public TestingSetup { * be used in "hot loops", for example fuzzing or benchmarking. */ template <class T = const BasicTestingSetup> -std::unique_ptr<T> MakeNoLogFileContext(const std::string& chain_name = CBaseChainParams::REGTEST, const std::vector<const char*>& extra_args = {}) +std::unique_ptr<T> MakeNoLogFileContext(const ChainType chain_type = ChainType::REGTEST, const std::vector<const char*>& extra_args = {}) { const std::vector<const char*> arguments = Cat( { @@ -215,7 +215,7 @@ std::unique_ptr<T> MakeNoLogFileContext(const std::string& chain_name = CBaseCha }, extra_args); - return std::make_unique<T>(chain_name, arguments); + return std::make_unique<T>(chain_type, arguments); } CBlock getBlock13b8a(); diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 1873cf5ec8..4797d9c310 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -22,8 +22,8 @@ CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) // chainparams.DefaultConsistencyChecks for tests .check_ratio = 1, }; - const auto err{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; - Assert(!err); + const auto result{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; + Assert(result); return mempool_opts; } diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 812737429d..26677bfa55 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -1687,7 +1687,7 @@ BOOST_AUTO_TEST_CASE(message_hash) BOOST_AUTO_TEST_CASE(remove_prefix) { - BOOST_CHECK_EQUAL(RemovePrefix("./util/system.h", "./"), "util/system.h"); + BOOST_CHECK_EQUAL(RemovePrefix("./common/system.h", "./"), "common/system.h"); BOOST_CHECK_EQUAL(RemovePrefixView("foo", "foo"), ""); BOOST_CHECK_EQUAL(RemovePrefix("foo", "fo"), "o"); BOOST_CHECK_EQUAL(RemovePrefixView("foo", "f"), "oo"); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index edc7e4b70a..b797de46af 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -4,6 +4,7 @@ // #include <chainparams.h> #include <consensus/validation.h> +#include <node/kernel_notifications.h> #include <node/utxo_snapshot.h> #include <random.h> #include <rpc/blockchain.h> @@ -22,6 +23,8 @@ #include <boost/test/unit_test.hpp> +using node::BlockManager; +using node::KernelNotifications; using node::SnapshotMetadata; BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, ChainTestingSetup) @@ -181,7 +184,7 @@ struct SnapshotTestSetup : TestChain100Setup { { LOCK(::cs_main); BOOST_CHECK(!chainman.IsSnapshotValidated()); - BOOST_CHECK(!node::FindSnapshotChainstateDir()); + BOOST_CHECK(!node::FindSnapshotChainstateDir(chainman.m_options.datadir)); } size_t initial_size; @@ -231,7 +234,7 @@ struct SnapshotTestSetup : TestChain100Setup { auto_infile >> coin; })); - BOOST_CHECK(!node::FindSnapshotChainstateDir()); + BOOST_CHECK(!node::FindSnapshotChainstateDir(chainman.m_options.datadir)); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { @@ -255,7 +258,7 @@ struct SnapshotTestSetup : TestChain100Setup { })); BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(this)); - BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir())); + BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir(chainman.m_options.datadir))); // Ensure our active chain is the snapshot chainstate. BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); @@ -268,7 +271,7 @@ struct SnapshotTestSetup : TestChain100Setup { { LOCK(::cs_main); - fs::path found = *node::FindSnapshotChainstateDir(); + fs::path found = *node::FindSnapshotChainstateDir(chainman.m_options.datadir); // Note: WriteSnapshotBaseBlockhash() is implicitly tested above. BOOST_CHECK_EQUAL( @@ -376,15 +379,21 @@ struct SnapshotTestSetup : TestChain100Setup { LOCK(::cs_main); chainman.ResetChainstates(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); + m_node.notifications = std::make_unique<KernelNotifications>(); const ChainstateManager::Options chainman_opts{ .chainparams = ::Params(), - .datadir = m_args.GetDataDirNet(), + .datadir = chainman.m_options.datadir, .adjusted_time_callback = GetAdjustedTime, + .notifications = *m_node.notifications, + }; + const BlockManager::Options blockman_opts{ + .chainparams = chainman_opts.chainparams, + .blocks_dir = m_args.GetBlocksDirPath(), }; // For robustness, ensure the old manager is destroyed before creating a // new one. m_node.chainman.reset(); - m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts, node::BlockManager::Options{}); + m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts, blockman_opts); } return *Assert(m_node.chainman); } @@ -482,7 +491,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup) this->SetupSnapshot(); - fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); + fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(chainman.m_options.datadir); BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); @@ -556,7 +565,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_completion, SnapshotTestSetup SnapshotCompletionResult res; auto mock_shutdown = [](bilingual_str msg) {}; - fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); + fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(chainman.m_options.datadir); BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index 0b4c491615..d00f2ff4d1 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -7,6 +7,7 @@ #include <net.h> #include <signet.h> #include <uint256.h> +#include <util/chaintype.h> #include <validation.h> #include <test/util/setup_common.h> @@ -41,7 +42,7 @@ static void TestBlockSubsidyHalvings(int nSubsidyHalvingInterval) BOOST_AUTO_TEST_CASE(block_subsidy_test) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); TestBlockSubsidyHalvings(chainParams->GetConsensus()); // As in main TestBlockSubsidyHalvings(150); // As in regtest TestBlockSubsidyHalvings(1000); // Just another interval @@ -49,7 +50,7 @@ BOOST_AUTO_TEST_CASE(block_subsidy_test) BOOST_AUTO_TEST_CASE(subsidy_limit_test) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); CAmount nSum = 0; for (int nHeight = 0; nHeight < 14000000; nHeight += 1000) { CAmount nSubsidy = GetBlockSubsidy(nHeight, chainParams->GetConsensus()); @@ -64,7 +65,7 @@ BOOST_AUTO_TEST_CASE(signet_parse_tests) { ArgsManager signet_argsman; signet_argsman.ForceSetArg("-signetchallenge", "51"); // set challenge to OP_TRUE - const auto signet_params = CreateChainParams(signet_argsman, CBaseChainParams::SIGNET); + const auto signet_params = CreateChainParams(signet_argsman, ChainType::SIGNET); CBlock block; BOOST_CHECK(signet_params->GetConsensus().signet_challenge == std::vector<uint8_t>{OP_TRUE}); CScript challenge{OP_TRUE}; @@ -124,7 +125,7 @@ BOOST_AUTO_TEST_CASE(signet_parse_tests) //! Test retrieval of valid assumeutxo values. BOOST_AUTO_TEST_CASE(test_assumeutxo) { - const auto params = CreateChainParams(*m_node.args, CBaseChainParams::REGTEST); + const auto params = CreateChainParams(*m_node.args, ChainType::REGTEST); // These heights don't have assumeutxo configurations associated, per the contents // of kernel/chainparams.cpp. diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 80c00036e7..9e69992752 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -7,6 +7,7 @@ #include <consensus/params.h> #include <test/util/random.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <versionbits.h> #include <boost/test/unit_test.hpp> @@ -418,8 +419,8 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // check that any deployment on any chain can conceivably reach both // ACTIVE and FAILED states in roughly the way we expect - for (const auto& chain_name : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET, CBaseChainParams::REGTEST}) { - const auto chainParams = CreateChainParams(*m_node.args, chain_name); + for (const auto& chain_type: {ChainType::MAIN, ChainType::TESTNET, ChainType::SIGNET, ChainType::REGTEST}) { + const auto chainParams = CreateChainParams(*m_node.args, chain_type); uint32_t chain_all_vbits{0}; for (int i = 0; i < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++i) { const auto dep = static_cast<Consensus::DeploymentPos>(i); @@ -440,7 +441,7 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // deployment that's not always/never active ArgsManager args; args.ForceSetArg("-vbparams", "testdummy:1199145601:1230767999"); // January 1, 2008 - December 31, 2008 - const auto chainParams = CreateChainParams(args, CBaseChainParams::REGTEST); + const auto chainParams = CreateChainParams(args, ChainType::REGTEST); check_computeblockversion(vbcache, chainParams->GetConsensus(), Consensus::DEPLOYMENT_TESTDUMMY); } @@ -450,7 +451,7 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // live deployment ArgsManager args; args.ForceSetArg("-vbparams", "testdummy:1199145601:1230767999:403200"); // January 1, 2008 - December 31, 2008, min act height 403200 - const auto chainParams = CreateChainParams(args, CBaseChainParams::REGTEST); + const auto chainParams = CreateChainParams(args, ChainType::REGTEST); check_computeblockversion(vbcache, chainParams->GetConsensus(), Consensus::DEPLOYMENT_TESTDUMMY); } } diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 92b55f9fc4..98d68f93e9 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -133,15 +133,15 @@ bool TorControlConnection::Connect(const std::string& tor_control_center, const Disconnect(); } - CService control_service; - if (!Lookup(tor_control_center, control_service, 9051, fNameLookup)) { + const std::optional<CService> control_service{Lookup(tor_control_center, 9051, fNameLookup)}; + if (!control_service.has_value()) { LogPrintf("tor: Failed to look up control center %s\n", tor_control_center); return false; } struct sockaddr_storage control_address; socklen_t control_address_len = sizeof(control_address); - if (!control_service.GetSockAddr(reinterpret_cast<struct sockaddr*>(&control_address), &control_address_len)) { + if (!control_service.value().GetSockAddr(reinterpret_cast<struct sockaddr*>(&control_address), &control_address_len)) { LogPrintf("tor: Error parsing socket address %s\n", tor_control_center); return false; } diff --git a/src/txdb.cpp b/src/txdb.cpp index 15351a4355..b2095bd4a3 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -31,22 +31,22 @@ static constexpr uint8_t DB_COINS{'c'}; static constexpr uint8_t DB_TXINDEX_BLOCK{'T'}; // uint8_t DB_TXINDEX{'t'} -std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db) +util::Result<void> CheckLegacyTxindex(CBlockTreeDB& block_tree_db) { CBlockLocator ignored{}; if (block_tree_db.Read(DB_TXINDEX_BLOCK, ignored)) { - return _("The -txindex upgrade started by a previous version cannot be completed. Restart with the previous version or run a full -reindex."); + return util::Error{_("The -txindex upgrade started by a previous version cannot be completed. Restart with the previous version or run a full -reindex.")}; } bool txindex_legacy_flag{false}; block_tree_db.ReadFlag("txindex", txindex_legacy_flag); if (txindex_legacy_flag) { // Disable legacy txindex and warn once about occupied disk space if (!block_tree_db.WriteFlag("txindex", false)) { - return Untranslated("Failed to write block index db flag 'txindex'='0'"); + return util::Error{Untranslated("Failed to write block index db flag 'txindex'='0'")}; } - return _("The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again."); + return util::Error{_("The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again.")}; } - return std::nullopt; + return {}; } bool CCoinsViewDB::NeedsUpgrade() diff --git a/src/txdb.h b/src/txdb.h index 63c7152097..04d0ecb39f 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -11,7 +11,11 @@ #include <kernel/cs_main.h> #include <sync.h> #include <util/fs.h> +#include <util/result.h> +#include <cstddef> +#include <cstdint> +#include <functional> #include <memory> #include <optional> #include <string> @@ -20,11 +24,11 @@ class CBlockFileInfo; class CBlockIndex; +class COutPoint; class uint256; namespace Consensus { struct Params; }; -struct bilingual_str; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; @@ -98,6 +102,6 @@ public: EXCLUSIVE_LOCKS_REQUIRED(::cs_main); }; -std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db); +[[nodiscard]] util::Result<void> CheckLegacyTxindex(CBlockTreeDB& block_tree_db); #endif // BITCOIN_TXDB_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index da875c271e..1286eba035 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -7,6 +7,7 @@ #include <chain.h> #include <coins.h> +#include <common/system.h> #include <consensus/consensus.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> @@ -19,13 +20,13 @@ #include <util/moneystr.h> #include <util/overflow.h> #include <util/result.h> -#include <util/system.h> #include <util/time.h> #include <util/trace.h> #include <util/translation.h> #include <validationinterface.h> #include <cmath> +#include <numeric> #include <optional> #include <string_view> #include <utility> @@ -74,7 +75,7 @@ void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap& cachedDescendan } // descendants now contains all in-mempool descendants of updateIt. // Update and add to cached descendant map - int64_t modifySize = 0; + int32_t modifySize = 0; CAmount modifyFee = 0; int64_t modifyCount = 0; for (const CTxMemPoolEntry& descendant : descendants) { @@ -90,7 +91,7 @@ void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap& cachedDescendan // Don't directly remove the transaction here -- doing so would // invalidate iterators in cachedDescendants. Mark it for removal // by inserting into descendants_to_remove. - if (descendant.GetCountWithAncestors() > uint64_t(m_limits.ancestor_count) || descendant.GetSizeWithAncestors() > uint64_t(m_limits.ancestor_size_vbytes)) { + if (descendant.GetCountWithAncestors() > uint64_t(m_limits.ancestor_count) || descendant.GetSizeWithAncestors() > m_limits.ancestor_size_vbytes) { descendants_to_remove.insert(descendant.GetTx().GetHash()); } } @@ -277,8 +278,8 @@ void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors for (const CTxMemPoolEntry& parent : parents) { UpdateChild(mapTx.iterator_to(parent), it, add); } - const int64_t updateCount = (add ? 1 : -1); - const int64_t updateSize = updateCount * it->GetTxSize(); + const int32_t updateCount = (add ? 1 : -1); + const int32_t updateSize{updateCount * it->GetTxSize()}; const CAmount updateFee = updateCount * it->GetModifiedFee(); for (txiter ancestorIt : setAncestors) { mapTx.modify(ancestorIt, [=](CTxMemPoolEntry& e) { e.UpdateDescendantState(updateSize, updateFee, updateCount); }); @@ -322,7 +323,7 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b setEntries setDescendants; CalculateDescendants(removeIt, setDescendants); setDescendants.erase(removeIt); // don't update state for self - int64_t modifySize = -((int64_t)removeIt->GetTxSize()); + int32_t modifySize = -removeIt->GetTxSize(); CAmount modifyFee = -removeIt->GetModifiedFee(); int modifySigOps = -removeIt->GetSigOpCost(); for (txiter dit : setDescendants) { @@ -364,21 +365,21 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b } } -void CTxMemPoolEntry::UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount) +void CTxMemPoolEntry::UpdateDescendantState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount) { nSizeWithDescendants += modifySize; - assert(int64_t(nSizeWithDescendants) > 0); + assert(nSizeWithDescendants > 0); nModFeesWithDescendants = SaturatingAdd(nModFeesWithDescendants, modifyFee); - nCountWithDescendants += modifyCount; + nCountWithDescendants += uint64_t(modifyCount); assert(int64_t(nCountWithDescendants) > 0); } -void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps) +void CTxMemPoolEntry::UpdateAncestorState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps) { nSizeWithAncestors += modifySize; - assert(int64_t(nSizeWithAncestors) > 0); + assert(nSizeWithAncestors > 0); nModFeesWithAncestors = SaturatingAdd(nModFeesWithAncestors, modifyFee); - nCountWithAncestors += modifyCount; + nCountWithAncestors += uint64_t(modifyCount); assert(int64_t(nCountWithAncestors) > 0); nSigOpCostWithAncestors += modifySigOps; assert(int(nSigOpCostWithAncestors) >= 0); @@ -698,7 +699,7 @@ void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendhei // Verify ancestor state is correct. auto ancestors{AssumeCalculateMemPoolAncestors(__func__, *it, Limits::NoLimits())}; uint64_t nCountCheck = ancestors.size() + 1; - uint64_t nSizeCheck = it->GetTxSize(); + int32_t nSizeCheck = it->GetTxSize(); CAmount nFeesCheck = it->GetModifiedFee(); int64_t nSigOpCheck = it->GetSigOpCost(); @@ -719,7 +720,7 @@ void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendhei // Check children against mapNextTx CTxMemPoolEntry::Children setChildrenCheck; auto iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0)); - uint64_t child_sizes = 0; + int32_t child_sizes{0}; for (; iter != mapNextTx.end() && iter->first->hash == it->GetTx().GetHash(); ++iter) { txiter childit = mapTx.find(iter->second->GetHash()); assert(childit != mapTx.end()); // mapNextTx points to in-mempool transactions @@ -755,11 +756,16 @@ void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendhei bool CTxMemPool::CompareDepthAndScore(const uint256& hasha, const uint256& hashb, bool wtxid) { + /* Return `true` if hasha should be considered sooner than hashb. Namely when: + * a is not in the mempool, but b is + * both are in the mempool and a has fewer ancestors than b + * both are in the mempool and a has a higher score than b + */ LOCK(cs); - indexed_transaction_set::const_iterator i = wtxid ? get_iter_from_wtxid(hasha) : mapTx.find(hasha); - if (i == mapTx.end()) return false; indexed_transaction_set::const_iterator j = wtxid ? get_iter_from_wtxid(hashb) : mapTx.find(hashb); - if (j == mapTx.end()) return true; + if (j == mapTx.end()) return false; + indexed_transaction_set::const_iterator i = wtxid ? get_iter_from_wtxid(hasha) : mapTx.find(hasha); + if (i == mapTx.end()) return true; uint64_t counta = i->GetCountWithAncestors(); uint64_t countb = j->GetCountWithAncestors(); if (counta == countb) { @@ -870,8 +876,17 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD } ++nTransactionsUpdated; } + if (delta == 0) { + mapDeltas.erase(hash); + LogPrintf("PrioritiseTransaction: %s (%sin mempool) delta cleared\n", hash.ToString(), it == mapTx.end() ? "not " : ""); + } else { + LogPrintf("PrioritiseTransaction: %s (%sin mempool) fee += %s, new delta=%s\n", + hash.ToString(), + it == mapTx.end() ? "not " : "", + FormatMoney(nFeeDelta), + FormatMoney(delta)); + } } - LogPrintf("PrioritiseTransaction: %s fee += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); } void CTxMemPool::ApplyDelta(const uint256& hash, CAmount &nFeeDelta) const @@ -890,6 +905,22 @@ void CTxMemPool::ClearPrioritisation(const uint256& hash) mapDeltas.erase(hash); } +std::vector<CTxMemPool::delta_info> CTxMemPool::GetPrioritisedTransactions() const +{ + AssertLockNotHeld(cs); + LOCK(cs); + std::vector<delta_info> result; + result.reserve(mapDeltas.size()); + for (const auto& [txid, delta] : mapDeltas) { + const auto iter{mapTx.find(txid)}; + const bool in_mempool{iter != mapTx.end()}; + std::optional<CAmount> modified_fee; + if (in_mempool) modified_fee = iter->GetModifiedFee(); + result.emplace_back(delta_info{in_mempool, delta, modified_fee, txid}); + } + return result; +} + const CTransaction* CTxMemPool::GetConflictTx(const COutPoint& prevout) const { const auto it = mapNextTx.find(prevout); @@ -913,6 +944,19 @@ CTxMemPool::setEntries CTxMemPool::GetIterSet(const std::set<uint256>& hashes) c return ret; } +std::vector<CTxMemPool::txiter> CTxMemPool::GetIterVec(const std::vector<uint256>& txids) const +{ + AssertLockHeld(cs); + std::vector<txiter> ret; + ret.reserve(txids.size()); + for (const auto& txid : txids) { + const auto it{GetIter(txid)}; + if (!it) return {}; + ret.push_back(*it); + } + return ret; +} + bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const { for (unsigned int i = 0; i < tx.vin.size(); i++) @@ -1142,7 +1186,6 @@ void CTxMemPool::SetLoadTried(bool load_tried) m_load_tried = load_tried; } - std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept { switch (r) { @@ -1155,3 +1198,30 @@ std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept } assert(false); } + +std::vector<CTxMemPool::txiter> CTxMemPool::GatherClusters(const std::vector<uint256>& txids) const +{ + AssertLockHeld(cs); + std::vector<txiter> clustered_txs{GetIterVec(txids)}; + // Use epoch: visiting an entry means we have added it to the clustered_txs vector. It does not + // necessarily mean the entry has been processed. + WITH_FRESH_EPOCH(m_epoch); + for (const auto& it : clustered_txs) { + visited(it); + } + // i = index of where the list of entries to process starts + for (size_t i{0}; i < clustered_txs.size(); ++i) { + // DoS protection: if there are 500 or more entries to process, just quit. + if (clustered_txs.size() > 500) return {}; + const txiter& tx_iter = clustered_txs.at(i); + for (const auto& entries : {tx_iter->GetMemPoolParentsConst(), tx_iter->GetMemPoolChildrenConst()}) { + for (const CTxMemPoolEntry& entry : entries) { + const auto entry_it = mapTx.iterator_to(entry); + if (!visited(entry_it)) { + clustered_txs.push_back(entry_it); + } + } + } + } + return clustered_txs; +} diff --git a/src/txmempool.h b/src/txmempool.h index 2c3cb7e9db..846def02cd 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -33,8 +33,11 @@ #include <util/result.h> #include <boost/multi_index/hashed_index.hpp> +#include <boost/multi_index/identity.hpp> +#include <boost/multi_index/indexed_by.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> +#include <boost/multi_index/tag.hpp> #include <boost/multi_index_container.hpp> class CBlockIndex; @@ -219,7 +222,7 @@ struct TxMempoolInfo CAmount fee; /** Virtual size of the transaction. */ - size_t vsize; + int32_t vsize; /** The fee delta. */ int64_t nFeeDelta; @@ -516,15 +519,35 @@ public: void ApplyDelta(const uint256& hash, CAmount &nFeeDelta) const EXCLUSIVE_LOCKS_REQUIRED(cs); void ClearPrioritisation(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs); + struct delta_info { + /** Whether this transaction is in the mempool. */ + const bool in_mempool; + /** The fee delta added using PrioritiseTransaction(). */ + const CAmount delta; + /** The modified fee (base fee + delta) of this entry. Only present if in_mempool=true. */ + std::optional<CAmount> modified_fee; + /** The prioritised transaction's txid. */ + const uint256 txid; + }; + /** Return a vector of all entries in mapDeltas with their corresponding delta_info. */ + std::vector<delta_info> GetPrioritisedTransactions() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + /** Get the transaction in the pool that spends the same prevout */ const CTransaction* GetConflictTx(const COutPoint& prevout) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Returns an iterator to the given hash, if found */ std::optional<txiter> GetIter(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs); - /** Translate a set of hashes into a set of pool iterators to avoid repeated lookups */ + /** Translate a set of hashes into a set of pool iterators to avoid repeated lookups. + * Does not require that all of the hashes correspond to actual transactions in the mempool, + * only returns the ones that exist. */ setEntries GetIterSet(const std::set<uint256>& hashes) const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** Translate a list of hashes into a list of mempool iterators to avoid repeated lookups. + * The nth element in txids becomes the nth element in the returned vector. If any of the txids + * don't actually exist in the mempool, returns an empty vector. */ + std::vector<txiter> GetIterVec(const std::vector<uint256>& txids) const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** Remove a set of transactions from the mempool. * If a transaction is in this set, then all in-mempool descendants must * also be in the set, unless this transaction is being removed for being @@ -585,6 +608,12 @@ public: const Limits& limits, bool fSearchForParents = true) const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** Collect the entire cluster of connected transactions for each transaction in txids. + * All txids must correspond to transaction entries in the mempool, otherwise this returns an + * empty vector. This call will also exit early and return an empty vector if it collects 500 or + * more transactions as a DoS protection. */ + std::vector<txiter> GatherClusters(const std::vector<uint256>& txids) const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** Calculate all in-mempool ancestors of a set of transactions not already in the mempool and * check ancestor and descendant limits. Heuristics are used to estimate the ancestor and * descendant count of all entries if the package were to be added to the mempool. The limits diff --git a/src/txrequest.cpp b/src/txrequest.cpp index 96a3d2eeeb..dd042103bd 100644 --- a/src/txrequest.cpp +++ b/src/txrequest.cpp @@ -10,8 +10,12 @@ #include <random.h> #include <uint256.h> -#include <boost/multi_index_container.hpp> +#include <boost/multi_index/indexed_by.hpp> #include <boost/multi_index/ordered_index.hpp> +#include <boost/multi_index/sequenced_index.hpp> +#include <boost/multi_index/tag.hpp> +#include <boost/multi_index_container.hpp> +#include <boost/tuple/tuple.hpp> #include <chrono> #include <unordered_map> @@ -69,7 +73,7 @@ struct Announcement { const bool m_is_wtxid : 1; /** What state this announcement is in. - * This is a uint8_t instead of a State to silence a GCC warning in versions prior to 8.4 and 9.3. + * This is a uint8_t instead of a State to silence a GCC warning in versions prior to 9.3. * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414 */ uint8_t m_state : 3; diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index d501c3fb69..004135ef97 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -123,7 +123,7 @@ public: const UniValue& get_array() const; enum VType type() const { return getType(); } - friend const UniValue& find_value( const UniValue& obj, const std::string& name); + const UniValue& find_value(std::string_view key) const; }; template <class It> @@ -201,6 +201,4 @@ static inline bool json_isspace(int ch) extern const UniValue NullUniValue; -const UniValue& find_value( const UniValue& obj, const std::string& name); - #endif // BITCOIN_UNIVALUE_INCLUDE_UNIVALUE_H diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 5aa39edb75..c3d19caae0 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -230,12 +230,13 @@ const char *uvTypeName(UniValue::VType t) return nullptr; } -const UniValue& find_value(const UniValue& obj, const std::string& name) +const UniValue& UniValue::find_value(std::string_view key) const { - for (unsigned int i = 0; i < obj.keys.size(); i++) - if (obj.keys[i] == name) - return obj.values.at(i); - + for (unsigned int i = 0; i < keys.size(); ++i) { + if (keys[i] == key) { + return values.at(i); + } + } return NullUniValue; } diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 5ddf300393..5fb973c67b 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -412,6 +412,33 @@ void univalue_readwrite() BOOST_CHECK_EQUAL(strJson1, v.write()); + // Valid + BOOST_CHECK(v.read("1.0") && (v.get_real() == 1.0)); + BOOST_CHECK(v.read("true") && v.get_bool()); + BOOST_CHECK(v.read("[false]") && !v[0].get_bool()); + BOOST_CHECK(v.read("{\"a\": true}") && v["a"].get_bool()); + BOOST_CHECK(v.read("{\"1\": \"true\"}") && (v["1"].get_str() == "true")); + // Valid, with leading or trailing whitespace + BOOST_CHECK(v.read(" 1.0") && (v.get_real() == 1.0)); + BOOST_CHECK(v.read("1.0 ") && (v.get_real() == 1.0)); + BOOST_CHECK(v.read("0.00000000000000000000000000000000000001e+30 ") && v.get_real() == 1e-8); + + BOOST_CHECK(!v.read(".19e-6")); //should fail, missing leading 0, therefore invalid JSON + // Invalid, initial garbage + BOOST_CHECK(!v.read("[1.0")); + BOOST_CHECK(!v.read("a1.0")); + // Invalid, trailing garbage + BOOST_CHECK(!v.read("1.0sds")); + BOOST_CHECK(!v.read("1.0]")); + // Invalid, keys have to be names + BOOST_CHECK(!v.read("{1: \"true\"}")); + BOOST_CHECK(!v.read("{true: 1}")); + BOOST_CHECK(!v.read("{[1]: 1}")); + BOOST_CHECK(!v.read("{{\"a\": \"a\"}: 1}")); + // BTC addresses should fail parsing + BOOST_CHECK(!v.read("175tWpb8K1S7NmH4Zx6rewF9WQrcZv245W")); + BOOST_CHECK(!v.read("3J98t1WpEZ73CNmQviecrnyiWrnqRhWNL")); + /* Check for (correctly reporting) a parsing error if the initial JSON construct is followed by more stuff. Note that whitespace is, of course, exempt. */ diff --git a/src/util/any.h b/src/util/any.h new file mode 100644 index 0000000000..4562c5bd8a --- /dev/null +++ b/src/util/any.h @@ -0,0 +1,26 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_ANY_H +#define BITCOIN_UTIL_ANY_H + +#include <any> + +namespace util { + +/** + * Helper function to access the contained object of a std::any instance. + * Returns a pointer to the object if passed instance has a value and the type + * matches, nullptr otherwise. + */ +template<typename T> +T* AnyPtr(const std::any& any) noexcept +{ + T* const* ptr = std::any_cast<T*>(&any); + return ptr ? *ptr : nullptr; +} + +} // namespace util + +#endif // BITCOIN_UTIL_ANY_H diff --git a/src/util/batchpriority.cpp b/src/util/batchpriority.cpp new file mode 100644 index 0000000000..c73aef1eb4 --- /dev/null +++ b/src/util/batchpriority.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <logging.h> +#include <util/syserror.h> + +#if (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) +#include <pthread.h> +#include <pthread_np.h> +#endif + +#ifndef WIN32 +#include <sched.h> +#endif + +void ScheduleBatchPriority() +{ +#ifdef SCHED_BATCH + const static sched_param param{}; + const int rc = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); + if (rc != 0) { + LogPrintf("Failed to pthread_setschedparam: %s\n", SysErrorString(rc)); + } +#endif +} diff --git a/src/util/batchpriority.h b/src/util/batchpriority.h new file mode 100644 index 0000000000..5ffc8dd684 --- /dev/null +++ b/src/util/batchpriority.h @@ -0,0 +1,15 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_BATCHPRIORITY_H +#define BITCOIN_UTIL_BATCHPRIORITY_H + +/** + * On platforms that support it, tell the kernel the calling thread is + * CPU-intensive and non-interactive. See SCHED_BATCH in sched(7) for details. + * + */ +void ScheduleBatchPriority(); + +#endif // BITCOIN_UTIL_BATCHPRIORITY_H diff --git a/src/util/bip32.cpp b/src/util/bip32.cpp index c0ad9257ce..4ab14bd704 100644 --- a/src/util/bip32.cpp +++ b/src/util/bip32.cpp @@ -51,17 +51,17 @@ bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypa return true; } -std::string FormatHDKeypath(const std::vector<uint32_t>& path) +std::string FormatHDKeypath(const std::vector<uint32_t>& path, bool apostrophe) { std::string ret; for (auto i : path) { ret += strprintf("/%i", (i << 1) >> 1); - if (i >> 31) ret += '\''; + if (i >> 31) ret += apostrophe ? '\'' : 'h'; } return ret; } -std::string WriteHDKeypath(const std::vector<uint32_t>& keypath) +std::string WriteHDKeypath(const std::vector<uint32_t>& keypath, bool apostrophe) { - return "m" + FormatHDKeypath(keypath); + return "m" + FormatHDKeypath(keypath, apostrophe); } diff --git a/src/util/bip32.h b/src/util/bip32.h index b720cb5638..ea5f192c07 100644 --- a/src/util/bip32.h +++ b/src/util/bip32.h @@ -13,7 +13,7 @@ [[nodiscard]] bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath); /** Write HD keypaths as strings */ -std::string WriteHDKeypath(const std::vector<uint32_t>& keypath); -std::string FormatHDKeypath(const std::vector<uint32_t>& path); +std::string WriteHDKeypath(const std::vector<uint32_t>& keypath, bool apostrophe = false); +std::string FormatHDKeypath(const std::vector<uint32_t>& path, bool apostrophe = false); #endif // BITCOIN_UTIL_BIP32_H diff --git a/src/util/chaintype.cpp b/src/util/chaintype.cpp new file mode 100644 index 0000000000..8a199e352a --- /dev/null +++ b/src/util/chaintype.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/chaintype.h> + +#include <cassert> +#include <optional> +#include <string> + +std::string ChainTypeToString(ChainType chain) +{ + switch (chain) { + case ChainType::MAIN: + return "main"; + case ChainType::TESTNET: + return "test"; + case ChainType::SIGNET: + return "signet"; + case ChainType::REGTEST: + return "regtest"; + } + assert(false); +} + +std::optional<ChainType> ChainTypeFromString(std::string_view chain) +{ + if (chain == "main") { + return ChainType::MAIN; + } else if (chain == "test") { + return ChainType::TESTNET; + } else if (chain == "signet") { + return ChainType::SIGNET; + } else if (chain == "regtest") { + return ChainType::REGTEST; + } else { + return std::nullopt; + } +} diff --git a/src/util/chaintype.h b/src/util/chaintype.h new file mode 100644 index 0000000000..c73985df57 --- /dev/null +++ b/src/util/chaintype.h @@ -0,0 +1,22 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_CHAINTYPE_H +#define BITCOIN_UTIL_CHAINTYPE_H + +#include <optional> +#include <string> + +enum class ChainType { + MAIN, + TESTNET, + SIGNET, + REGTEST, +}; + +std::string ChainTypeToString(ChainType chain); + +std::optional<ChainType> ChainTypeFromString(std::string_view chain); + +#endif // BITCOIN_UTIL_CHAINTYPE_H diff --git a/src/util/fs.h b/src/util/fs.h index 886a30394e..8f79f6cba6 100644 --- a/src/util/fs.h +++ b/src/util/fs.h @@ -8,7 +8,7 @@ #include <tinyformat.h> #include <cstdio> -#include <filesystem> +#include <filesystem> // IWYU pragma: export #include <functional> #include <iomanip> #include <ios> diff --git a/src/util/insert.h b/src/util/insert.h new file mode 100644 index 0000000000..5332eca60a --- /dev/null +++ b/src/util/insert.h @@ -0,0 +1,24 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_INSERT_H +#define BITCOIN_UTIL_INSERT_H + +#include <set> + +namespace util { + +//! Simplification of std insertion +template <typename Tdst, typename Tsrc> +inline void insert(Tdst& dst, const Tsrc& src) { + dst.insert(dst.begin(), src.begin(), src.end()); +} +template <typename TsetT, typename Tsrc> +inline void insert(std::set<TsetT>& dst, const Tsrc& src) { + dst.insert(src.begin(), src.end()); +} + +} // namespace util + +#endif // BITCOIN_UTIL_INSERT_H diff --git a/src/util/result.h b/src/util/result.h index 972b1aada0..b99995c7e5 100644 --- a/src/util/result.h +++ b/src/util/result.h @@ -31,16 +31,19 @@ struct Error { //! `std::optional<T>` can be updated to return `util::Result<T>` and return //! error strings usually just replacing `return std::nullopt;` with `return //! util::Error{error_string};`. -template <class T> +template <class M> class Result { private: + using T = std::conditional_t<std::is_same_v<M, void>, std::monostate, M>; + std::variant<bilingual_str, T> m_variant; template <typename FT> friend bilingual_str ErrorString(const Result<FT>& result); public: + Result() : m_variant{std::in_place_index_t<1>{}, std::monostate{}} {} // constructor for void Result(T obj) : m_variant{std::in_place_index_t<1>{}, std::move(obj)} {} Result(Error error) : m_variant{std::in_place_index_t<0>{}, std::move(error.message)} {} diff --git a/src/util/sock.cpp b/src/util/sock.cpp index 53d20bdf19..c83869bc77 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -2,12 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <compat/compat.h> #include <logging.h> #include <tinyformat.h> #include <util/sock.h> #include <util/syserror.h> -#include <util/system.h> #include <util/threadinterrupt.h> #include <util/time.h> diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 05e7b957c4..d792562735 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -17,8 +17,8 @@ #include <cstdint> #include <limits> #include <optional> -#include <string> -#include <string_view> +#include <string> // IWYU pragma: export +#include <string_view> // IWYU pragma: export #include <system_error> #include <type_traits> #include <vector> diff --git a/src/util/string.h b/src/util/string.h index fb93d2a80e..8b69d6ccae 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -12,8 +12,8 @@ #include <cstring> #include <locale> #include <sstream> -#include <string> -#include <string_view> +#include <string> // IWYU pragma: export +#include <string_view> // IWYU pragma: export #include <vector> void ReplaceAll(std::string& in_out, const std::string& search, const std::string& substitute); diff --git a/src/util/system.h b/src/util/system.h deleted file mode 100644 index e2fc3450f6..0000000000 --- a/src/util/system.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_SYSTEM_H -#define BITCOIN_UTIL_SYSTEM_H - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <compat/assumptions.h> -#include <compat/compat.h> - -#include <any> -#include <set> -#include <stdint.h> -#include <string> - -// Application startup time (used for uptime calculation) -int64_t GetStartupTime(); - -void SetupEnvironment(); -bool SetupNetworking(); -#ifndef WIN32 -std::string ShellEscape(const std::string& arg); -#endif -#if HAVE_SYSTEM -void runCommand(const std::string& strCommand); -#endif - -/** - * Return the number of cores available on the current system. - * @note This does count virtual cores, such as those provided by HyperThreading. - */ -int GetNumCores(); - -/** - * On platforms that support it, tell the kernel the calling thread is - * CPU-intensive and non-interactive. See SCHED_BATCH in sched(7) for details. - * - */ -void ScheduleBatchPriority(); - -namespace util { - -//! Simplification of std insertion -template <typename Tdst, typename Tsrc> -inline void insert(Tdst& dst, const Tsrc& src) { - dst.insert(dst.begin(), src.begin(), src.end()); -} -template <typename TsetT, typename Tsrc> -inline void insert(std::set<TsetT>& dst, const Tsrc& src) { - dst.insert(src.begin(), src.end()); -} - -/** - * Helper function to access the contained object of a std::any instance. - * Returns a pointer to the object if passed instance has a value and the type - * matches, nullptr otherwise. - */ -template<typename T> -T* AnyPtr(const std::any& any) noexcept -{ - T* const* ptr = std::any_cast<T*>(&any); - return ptr ? *ptr : nullptr; -} - -} // namespace util - -#endif // BITCOIN_UTIL_SYSTEM_H diff --git a/src/util/time.cpp b/src/util/time.cpp index fb9bc34931..5ca9d21f8d 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -78,14 +78,6 @@ NodeClock::time_point NodeClock::now() noexcept return time_point{ret}; }; -template <typename T> -static T GetSystemTime() -{ - const auto now = std::chrono::duration_cast<T>(std::chrono::system_clock::now().time_since_epoch()); - assert(now.count() > 0); - return now; -} - void SetMockTime(int64_t nMockTimeIn) { Assert(nMockTimeIn >= 0); @@ -102,11 +94,6 @@ std::chrono::seconds GetMockTime() return std::chrono::seconds(nMockTime.load(std::memory_order_relaxed)); } -int64_t GetTimeMillis() -{ - return int64_t{GetSystemTime<std::chrono::milliseconds>().count()}; -} - int64_t GetTime() { return GetTime<std::chrono::seconds>().count(); } std::string FormatISO8601DateTime(int64_t nTime) { diff --git a/src/util/time.h b/src/util/time.h index 8c6baeb12a..b6aab615ba 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -71,9 +71,6 @@ using MillisecondsDouble = std::chrono::duration<double, std::chrono::millisecon */ int64_t GetTime(); -/** Returns the system time (not mockable) */ -int64_t GetTimeMillis(); - /** * DEPRECATED * Use SetMockTime with chrono type diff --git a/src/validation.cpp b/src/validation.cpp index b2f4283e16..d9a0fce34f 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -11,7 +11,6 @@ #include <arith_uint256.h> #include <chain.h> #include <checkqueue.h> -#include <common/args.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> @@ -23,10 +22,10 @@ #include <hash.h> #include <kernel/chainparams.h> #include <kernel/mempool_entry.h> +#include <kernel/notifications_interface.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> -#include <node/interface_ui.h> #include <node/utxo_snapshot.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -52,7 +51,6 @@ #include <util/moneystr.h> #include <util/rbf.h> #include <util/strencodings.h> -#include <util/system.h> #include <util/time.h> #include <util/trace.h> #include <util/translation.h> @@ -72,6 +70,7 @@ using kernel::CCoinsStats; using kernel::CoinStatsHashType; using kernel::ComputeUTXOStats; using kernel::LoadMempool; +using kernel::Notifications; using fsbridge::FopenFn; using node::BlockManager; @@ -79,10 +78,7 @@ using node::BlockMap; using node::CBlockIndexHeightOnlyComparator; using node::CBlockIndexWorkComparator; using node::fReindex; -using node::ReadBlockFromDisk; using node::SnapshotMetadata; -using node::UndoReadFromDisk; -using node::UnlinkPrunedFiles; /** Maximum kilobytes for transactions to store for processing during reorg */ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; @@ -1642,26 +1638,6 @@ bool Chainstate::IsInitialBlockDownload() const return false; } -static void AlertNotify(const std::string& strMessage) -{ - uiInterface.NotifyAlertChanged(); -#if HAVE_SYSTEM - std::string strCmd = gArgs.GetArg("-alertnotify", ""); - if (strCmd.empty()) return; - - // Alert text should be plain ascii coming from a trusted source, but to - // be safe we first strip anything not in safeChars, then add single quotes around - // the whole string before passing it to the shell: - std::string singleQuote("'"); - std::string safeStatus = SanitizeString(strMessage); - safeStatus = singleQuote+safeStatus+singleQuote; - ReplaceAll(strCmd, "%s", safeStatus); - - std::thread t(runCommand, strCmd); - t.detach(); // thread runs free -#endif -} - void Chainstate::CheckForkWarningConditions() { AssertLockHeld(cs_main); @@ -1914,7 +1890,7 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn bool fClean = true; CBlockUndo blockUndo; - if (!UndoReadFromDisk(blockUndo, pindex)) { + if (!m_blockman.UndoReadFromDisk(blockUndo, *pindex)) { error("DisconnectBlock(): failure reading undo data"); return DISCONNECT_FAILED; } @@ -2017,8 +1993,6 @@ public: } }; -static std::array<ThresholdConditionCache, VERSIONBITS_NUM_BITS> warningcache GUARDED_BY(cs_main); - static unsigned int GetBlockScriptFlags(const CBlockIndex& block_index, const ChainstateManager& chainman) { const Consensus::Params& consensusparams = chainman.GetConsensus(); @@ -2374,7 +2348,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, if (fJustCheck) return true; - if (!m_blockman.WriteUndoDataForBlock(blockundo, state, pindex, params)) { + if (!m_blockman.WriteUndoDataForBlock(blockundo, state, *pindex)) { return false; } @@ -2523,7 +2497,7 @@ bool Chainstate::FlushStateToDisk( // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Ensure we can write block index - if (!CheckDiskSpace(gArgs.GetBlocksDirPath())) { + if (!CheckDiskSpace(m_blockman.m_opts.blocks_dir)) { return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } { @@ -2545,7 +2519,7 @@ bool Chainstate::FlushStateToDisk( if (fFlushForPrune) { LOG_TIME_MILLIS_WITH_CATEGORY("unlink pruned files", BCLog::BENCH); - UnlinkPrunedFiles(setFilesToPrune); + m_blockman.UnlinkPrunedFiles(setFilesToPrune); } m_last_write = nNow; } @@ -2559,7 +2533,7 @@ bool Chainstate::FlushStateToDisk( // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(gArgs.GetDataDirNet(), 48 * 2 * 2 * CoinsTip().GetCacheSize())) { + if (!CheckDiskSpace(m_chainman.m_options.datadir, 48 * 2 * 2 * CoinsTip().GetCacheSize())) { return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } // Flush the chainstate (which may refer to block index entries). @@ -2602,16 +2576,6 @@ void Chainstate::PruneAndFlush() } } -static void DoWarning(const bilingual_str& warning) -{ - static bool fWarned = false; - SetMiscWarning(warning); - if (!fWarned) { - AlertNotify(warning.original); - fWarned = true; - } -} - /** Private helper function that concatenates warning messages. */ static void AppendWarning(bilingual_str& res, const bilingual_str& warn) { @@ -2674,11 +2638,11 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(m_chainman, bit); - ThresholdState state = checker.GetStateFor(pindex, params.GetConsensus(), warningcache.at(bit)); + ThresholdState state = checker.GetStateFor(pindex, params.GetConsensus(), m_chainman.m_warningcache.at(bit)); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { - DoWarning(warning); + m_chainman.GetNotifications().warning(warning); } else { AppendWarning(warning_messages, warning); } @@ -2709,7 +2673,7 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra // Read block from disk. std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); CBlock& block = *pblock; - if (!ReadBlockFromDisk(block, pindexDelete, m_chainman.GetConsensus())) { + if (!m_blockman.ReadBlockFromDisk(block, *pindexDelete)) { return error("DisconnectTip(): Failed to read block"); } // Apply the block atomically to the chain state. @@ -2763,7 +2727,6 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra return true; } -static SteadyClock::duration time_read_from_disk_total{}; static SteadyClock::duration time_connect_total{}; static SteadyClock::duration time_flush{}; static SteadyClock::duration time_chainstate{}; @@ -2826,7 +2789,7 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, std::shared_ptr<const CBlock> pthisBlock; if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); - if (!ReadBlockFromDisk(*pblockNew, pindexNew, m_chainman.GetConsensus())) { + if (!m_blockman.ReadBlockFromDisk(*pblockNew, *pindexNew)) { return AbortNode(state, "Failed to read block"); } pthisBlock = pblockNew; @@ -2837,12 +2800,11 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const CBlock& blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. const auto time_2{SteadyClock::now()}; - time_read_from_disk_total += time_2 - time_1; SteadyClock::time_point time_3; - LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs (%.2fms/blk)]\n", - Ticks<MillisecondsDouble>(time_2 - time_1), - Ticks<SecondsDouble>(time_read_from_disk_total), - Ticks<MillisecondsDouble>(time_read_from_disk_total) / num_blocks_total); + // When adding aggregate statistics in the future, keep in mind that + // num_blocks_total may be zero until the ConnectBlock() call below. + LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms\n", + Ticks<MillisecondsDouble>(time_2 - time_1)); { CCoinsViewCache view(&CoinsTip()); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view); @@ -3100,7 +3062,7 @@ static bool NotifyHeaderTip(Chainstate& chainstate) LOCKS_EXCLUDED(cs_main) { } // Send block tip changed notifications without cs_main if (fNotify) { - uiInterface.NotifyHeaderTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader->nHeight, pindexHeader->nTime, false); + chainstate.m_chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader->nHeight, pindexHeader->nTime, false); } return fNotify; } @@ -3139,7 +3101,6 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< CBlockIndex *pindexMostWork = nullptr; CBlockIndex *pindexNewTip = nullptr; - int nStopAtHeight = gArgs.GetIntArg("-stopatheight", DEFAULT_STOPATHEIGHT); do { // Block until the validation queue drains. This should largely // never happen in normal operation, however may happen during @@ -3209,12 +3170,12 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); // Always notify the UI if a new block tip was connected - uiInterface.NotifyBlockTip(GetSynchronizationState(fInitialDownload), pindexNewTip); + m_chainman.GetNotifications().blockTip(GetSynchronizationState(fInitialDownload), *pindexNewTip); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). - if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) StartShutdown(); + if (m_chainman.StopAtHeight() && pindexNewTip && pindexNewTip->nHeight >= m_chainman.StopAtHeight()) StartShutdown(); if (WITH_LOCK(::cs_main, return m_disabled)) { // Background chainstate has reached the snapshot base block, so exit. @@ -3406,7 +3367,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde // Only notify about a new block tip if the active chain was modified. if (pindex_was_in_chain) { - uiInterface.NotifyBlockTip(GetSynchronizationState(IsInitialBlockDownload()), to_mark_failed->pprev); + m_chainman.GetNotifications().blockTip(GetSynchronizationState(IsInitialBlockDownload()), *to_mark_failed->pprev); } return true; } @@ -3923,7 +3884,7 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t m_last_presync_update = now; } bool initial_download = chainstate.IsInitialBlockDownload(); - uiInterface.NotifyHeaderTip(GetSynchronizationState(initial_download), height, timestamp, /*presync=*/true); + GetNotifications().headerTip(GetSynchronizationState(initial_download), height, timestamp, /*presync=*/true); if (initial_download) { const int64_t blocks_left{(GetTime() - timestamp) / GetConsensus().nPowTargetSpacing}; const double progress{100.0 * height / (height + blocks_left)}; @@ -4000,7 +3961,7 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV // Write block to history file if (fNewBlock) *fNewBlock = true; try { - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, params, dbp)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, dbp)}; if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -4148,14 +4109,15 @@ bool Chainstate::LoadChainTip() return true; } -CVerifyDB::CVerifyDB() +CVerifyDB::CVerifyDB(Notifications& notifications) + : m_notifications{notifications} { - uiInterface.ShowProgress(_("Verifying blocks…").translated, 0, false); + m_notifications.progress(_("Verifying blocks…"), 0, false); } CVerifyDB::~CVerifyDB() { - uiInterface.ShowProgress("", 100, false); + m_notifications.progress(bilingual_str{}, 100, false); } VerifyDBResult CVerifyDB::VerifyDB( @@ -4195,7 +4157,7 @@ VerifyDBResult CVerifyDB::VerifyDB( LogPrintf("Verification progress: %d%%\n", percentageDone); reportDone = percentageDone / 10; } - uiInterface.ShowProgress(_("Verifying blocks…").translated, percentageDone, false); + m_notifications.progress(_("Verifying blocks…"), percentageDone, false); if (pindex->nHeight <= chainstate.m_chain.Height() - nCheckDepth) { break; } @@ -4208,7 +4170,7 @@ VerifyDBResult CVerifyDB::VerifyDB( } CBlock block; // check level 0: read from disk - if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + if (!chainstate.m_blockman.ReadBlockFromDisk(block, *pindex)) { LogPrintf("Verification error: ReadBlockFromDisk failed at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); return VerifyDBResult::CORRUPTED_BLOCK_DB; } @@ -4222,7 +4184,7 @@ VerifyDBResult CVerifyDB::VerifyDB( if (nCheckLevel >= 2 && pindex) { CBlockUndo undo; if (!pindex->GetUndoPos().IsNull()) { - if (!UndoReadFromDisk(undo, pindex)) { + if (!chainstate.m_blockman.UndoReadFromDisk(undo, *pindex)) { LogPrintf("Verification error: found bad undo data at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); return VerifyDBResult::CORRUPTED_BLOCK_DB; } @@ -4271,10 +4233,10 @@ VerifyDBResult CVerifyDB::VerifyDB( LogPrintf("Verification progress: %d%%\n", percentageDone); reportDone = percentageDone / 10; } - uiInterface.ShowProgress(_("Verifying blocks…").translated, percentageDone, false); + m_notifications.progress(_("Verifying blocks…"), percentageDone, false); pindex = chainstate.m_chain.Next(pindex); CBlock block; - if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + if (!chainstate.m_blockman.ReadBlockFromDisk(block, *pindex)) { LogPrintf("Verification error: ReadBlockFromDisk failed at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); return VerifyDBResult::CORRUPTED_BLOCK_DB; } @@ -4303,7 +4265,7 @@ bool Chainstate::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& in AssertLockHeld(cs_main); // TODO: merge with ConnectBlock CBlock block; - if (!ReadBlockFromDisk(block, pindex, m_chainman.GetConsensus())) { + if (!m_blockman.ReadBlockFromDisk(block, *pindex)) { return error("ReplayBlock(): ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -4330,7 +4292,7 @@ bool Chainstate::ReplayBlocks() if (hashHeads.empty()) return true; // We're already in a consistent state. if (hashHeads.size() != 2) return error("ReplayBlocks(): unknown inconsistent state"); - uiInterface.ShowProgress(_("Replaying blocks…").translated, 0, false); + m_chainman.GetNotifications().progress(_("Replaying blocks…"), 0, false); LogPrintf("Replaying blocks\n"); const CBlockIndex* pindexOld = nullptr; // Old tip during the interrupted flush. @@ -4355,7 +4317,7 @@ bool Chainstate::ReplayBlocks() while (pindexOld != pindexFork) { if (pindexOld->nHeight > 0) { // Never disconnect the genesis block. CBlock block; - if (!ReadBlockFromDisk(block, pindexOld, m_chainman.GetConsensus())) { + if (!m_blockman.ReadBlockFromDisk(block, *pindexOld)) { return error("RollbackBlock(): ReadBlockFromDisk() failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); } LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); @@ -4377,13 +4339,13 @@ bool Chainstate::ReplayBlocks() const CBlockIndex& pindex{*Assert(pindexNew->GetAncestor(nHeight))}; LogPrintf("Rolling forward %s (%i)\n", pindex.GetBlockHash().ToString(), nHeight); - uiInterface.ShowProgress(_("Replaying blocks…").translated, (int) ((nHeight - nForkHeight) * 100.0 / (pindexNew->nHeight - nForkHeight)) , false); + m_chainman.GetNotifications().progress(_("Replaying blocks…"), (int)((nHeight - nForkHeight) * 100.0 / (pindexNew->nHeight - nForkHeight)), false); if (!RollforwardBlock(&pindex, cache)) return false; } cache.SetBestBlock(pindexNew->GetBlockHash()); cache.Flush(); - uiInterface.ShowProgress("", 100, false); + m_chainman.GetNotifications().progress(bilingual_str{}, 100, false); return true; } @@ -4418,7 +4380,7 @@ bool ChainstateManager::LoadBlockIndex() // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret = m_blockman.LoadBlockIndexDB(GetConsensus()); + bool ret{m_blockman.LoadBlockIndexDB()}; if (!ret) return false; m_blockman.ScanAndUnlinkAlreadyPrunedFiles(); @@ -4522,7 +4484,7 @@ bool Chainstate::LoadGenesisBlock() try { const CBlock& block = params.GenesisBlock(); - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, params, nullptr)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, nullptr)}; if (blockPos.IsNull()) { return error("%s: writing genesis block to disk failed", __func__); } @@ -4565,7 +4527,7 @@ void Chainstate::LoadExternalBlockFile( try { // locate a header unsigned char buf[CMessageHeader::MESSAGE_START_SIZE]; - blkdat.FindByte(params.MessageStart()[0]); + blkdat.FindByte(std::byte(params.MessageStart()[0])); nRewind = blkdat.GetPos() + 1; blkdat >> buf; if (memcmp(buf, params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) { @@ -4593,6 +4555,9 @@ void Chainstate::LoadExternalBlockFile( // next block, but it's still possible to rewind to the start of the current block (without a disk read). nRewind = nBlockPos + nSize; blkdat.SkipTo(nRewind); + + std::shared_ptr<CBlock> pblock{}; // needs to remain available after the cs_main lock is released to avoid duplicate reads from disk + { LOCK(cs_main); // detect out of order blocks, and store them for later @@ -4610,7 +4575,7 @@ void Chainstate::LoadExternalBlockFile( if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { // This block can be processed immediately; rewind to its start, read and deserialize it. blkdat.SetPos(nBlockPos); - std::shared_ptr<CBlock> pblock{std::make_shared<CBlock>()}; + pblock = std::make_shared<CBlock>(); blkdat >> *pblock; nRewind = blkdat.GetPos(); @@ -4634,6 +4599,21 @@ void Chainstate::LoadExternalBlockFile( } } + if (m_blockman.IsPruneMode() && !fReindex && pblock) { + // must update the tip for pruning to work while importing with -loadblock. + // this is a tradeoff to conserve disk space at the expense of time + // spent updating the tip to be able to prune. + // otherwise, ActivateBestChain won't be called by the import process + // until after all of the block files are loaded. ActivateBestChain can be + // called by concurrent network message processing. but, that is not + // reliable for the purpose of pruning while importing. + BlockValidationState state; + if (!ActivateBestChain(state, pblock)) { + LogPrint(BCLog::REINDEX, "failed to activate chain (%s)\n", state.ToString()); + break; + } + } + NotifyHeaderTip(*this); if (!blocks_with_unknown_parent) continue; @@ -4648,7 +4628,7 @@ void Chainstate::LoadExternalBlockFile( while (range.first != range.second) { std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); - if (ReadBlockFromDisk(*pblockrecursive, it->second, params.GetConsensus())) { + if (m_blockman.ReadBlockFromDisk(*pblockrecursive, it->second)) { LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); @@ -5124,7 +5104,7 @@ bool ChainstateManager::ActivateSnapshot( // PopulateAndValidateSnapshot can return (in error) before the leveldb datadir // has been created, so only attempt removal if we got that far. - if (auto snapshot_datadir = node::FindSnapshotChainstateDir()) { + if (auto snapshot_datadir = node::FindSnapshotChainstateDir(m_options.datadir)) { // We have to destruct leveldb::DB in order to release the db lock, otherwise // DestroyDB() (in DeleteCoinsDBFromDisk()) will fail. See `leveldb::~DBImpl()`. // Destructing the chainstate (and so resetting the coinsviews object) does this. @@ -5604,17 +5584,12 @@ ChainstateManager::~ChainstateManager() LOCK(::cs_main); m_versionbitscache.Clear(); - - // TODO: The warning cache should probably become non-global - for (auto& i : warningcache) { - i.clear(); - } } bool ChainstateManager::DetectSnapshotChainstate(CTxMemPool* mempool) { assert(!m_snapshot_chainstate); - std::optional<fs::path> path = node::FindSnapshotChainstateDir(); + std::optional<fs::path> path = node::FindSnapshotChainstateDir(m_options.datadir); if (!path) { return false; } diff --git a/src/validation.h b/src/validation.h index fd0e2115f7..cb5b291dab 100644 --- a/src/validation.h +++ b/src/validation.h @@ -65,8 +65,6 @@ struct Params; static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; -/** Default for -stopatheight */ -static const int DEFAULT_STOPATHEIGHT = 0; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ static const unsigned int MIN_BLOCKS_TO_KEEP = 288; static const signed int DEFAULT_CHECKBLOCKS = 6; @@ -364,9 +362,13 @@ enum class VerifyDBResult { }; /** RAII wrapper for VerifyDB: Verify consistency of the block and coin databases */ -class CVerifyDB { +class CVerifyDB +{ +private: + kernel::Notifications& m_notifications; + public: - CVerifyDB(); + explicit CVerifyDB(kernel::Notifications& notifications); ~CVerifyDB(); [[nodiscard]] VerifyDBResult VerifyDB( Chainstate& chainstate, @@ -936,6 +938,8 @@ private: //! nullopt. std::optional<int> GetSnapshotBaseHeight() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + std::array<ThresholdConditionCache, VERSIONBITS_NUM_BITS> m_warningcache GUARDED_BY(::cs_main); + //! Return true if a chainstate is considered usable. //! //! This is false when a background validation chainstate has completed its @@ -955,6 +959,8 @@ public: bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } const arith_uint256& MinimumChainWork() const { return *Assert(m_options.minimum_chain_work); } const uint256& AssumedValidBlock() const { return *Assert(m_options.assumed_valid_block); } + kernel::Notifications& GetNotifications() const { return m_options.notifications; }; + int StopAtHeight() const { return m_options.stop_at_height; }; /** * Alias for ::cs_main. diff --git a/src/version.h b/src/version.h index ee646eefc3..611a670314 100644 --- a/src/version.h +++ b/src/version.h @@ -20,9 +20,6 @@ static const int MIN_PEER_PROTO_VERSION = 31800; //! BIP 0031, pong message, is enabled for all versions AFTER this one static const int BIP0031_VERSION = 60000; -//! "filter*" commands are disabled without NODE_BLOOM after and including this version -static const int NO_BLOOM_VERSION = 70011; - //! "sendheaders" command and announcing blocks with headers starts with this version static const int SENDHEADERS_VERSION = 70012; diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index fa6e56c6e4..68abdcd81e 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -668,14 +668,15 @@ void BerkeleyDatabase::ReloadDbEnv() env->ReloadDbEnv(); } -BerkeleyCursor::BerkeleyCursor(BerkeleyDatabase& database, BerkeleyBatch* batch) +BerkeleyCursor::BerkeleyCursor(BerkeleyDatabase& database, const BerkeleyBatch& batch, Span<const std::byte> prefix) + : m_key_prefix(prefix.begin(), prefix.end()) { if (!database.m_db.get()) { throw std::runtime_error(STR_INTERNAL_BUG("BerkeleyDatabase does not exist")); } // Transaction argument to cursor is only needed when using the cursor to // write to the database. Read-only cursors do not need a txn pointer. - int ret = database.m_db->cursor(batch ? batch->txn() : nullptr, &m_cursor, 0); + int ret = database.m_db->cursor(batch.txn(), &m_cursor, 0); if (ret != 0) { throw std::runtime_error(STR_INTERNAL_BUG(strprintf("BDB Cursor could not be created. Returned %d", ret))); } @@ -685,19 +686,30 @@ DatabaseCursor::Status BerkeleyCursor::Next(DataStream& ssKey, DataStream& ssVal { if (m_cursor == nullptr) return Status::FAIL; // Read at cursor - SafeDbt datKey; + SafeDbt datKey(m_key_prefix.data(), m_key_prefix.size()); SafeDbt datValue; - int ret = m_cursor->get(datKey, datValue, DB_NEXT); + int ret = -1; + if (m_first && !m_key_prefix.empty()) { + ret = m_cursor->get(datKey, datValue, DB_SET_RANGE); + } else { + ret = m_cursor->get(datKey, datValue, DB_NEXT); + } + m_first = false; if (ret == DB_NOTFOUND) { return Status::DONE; } - if (ret != 0 || datKey.get_data() == nullptr || datValue.get_data() == nullptr) { + if (ret != 0) { return Status::FAIL; } + Span<const std::byte> raw_key = {AsBytePtr(datKey.get_data()), datKey.get_size()}; + if (!m_key_prefix.empty() && std::mismatch(raw_key.begin(), raw_key.end(), m_key_prefix.begin(), m_key_prefix.end()).second != m_key_prefix.end()) { + return Status::DONE; + } + // Convert to streams ssKey.clear(); - ssKey.write({AsBytePtr(datKey.get_data()), datKey.get_size()}); + ssKey.write(raw_key); ssValue.clear(); ssValue.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); return Status::MORE; @@ -713,7 +725,13 @@ BerkeleyCursor::~BerkeleyCursor() std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewCursor() { if (!pdb) return nullptr; - return std::make_unique<BerkeleyCursor>(m_database); + return std::make_unique<BerkeleyCursor>(m_database, *this); +} + +std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewPrefixCursor(Span<const std::byte> prefix) +{ + if (!pdb) return nullptr; + return std::make_unique<BerkeleyCursor>(m_database, *this, prefix); } bool BerkeleyBatch::TxnBegin() @@ -777,6 +795,7 @@ bool BerkeleyBatch::ReadKey(DataStream&& key, DataStream& value) SafeDbt datValue; int ret = pdb->get(activeTxn, datKey, datValue, 0); if (ret == 0 && datValue.get_data() != nullptr) { + value.clear(); value.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); return true; } @@ -825,7 +844,7 @@ bool BerkeleyBatch::HasKey(DataStream&& key) bool BerkeleyBatch::ErasePrefix(Span<const std::byte> prefix) { if (!TxnBegin()) return false; - auto cursor{std::make_unique<BerkeleyCursor>(m_database, this)}; + auto cursor{std::make_unique<BerkeleyCursor>(m_database, *this)}; // const_cast is safe below even though prefix_key is an in/out parameter, // because we are not using the DB_DBT_USERMEM flag, so BDB will allocate // and return a different output data pointer diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index 4fac128bf1..8cc03692d6 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -7,10 +7,10 @@ #define BITCOIN_WALLET_BDB_H #include <clientversion.h> +#include <common/system.h> #include <serialize.h> #include <streams.h> #include <util/fs.h> -#include <util/system.h> #include <wallet/db.h> #include <atomic> @@ -190,9 +190,13 @@ class BerkeleyCursor : public DatabaseCursor { private: Dbc* m_cursor; + std::vector<std::byte> m_key_prefix; + bool m_first{true}; public: - explicit BerkeleyCursor(BerkeleyDatabase& database, BerkeleyBatch* batch=nullptr); + // Constructor for cursor for records matching the prefix + // To match all records, an empty prefix may be provided. + explicit BerkeleyCursor(BerkeleyDatabase& database, const BerkeleyBatch& batch, Span<const std::byte> prefix = {}); ~BerkeleyCursor() override; Status Next(DataStream& key, DataStream& value) override; @@ -229,6 +233,7 @@ public: void Close() override; std::unique_ptr<DatabaseCursor> GetNewCursor() override; + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override; bool TxnBegin() override; bool TxnCommit() override; bool TxnAbort() override; diff --git a/src/wallet/coincontrol.cpp b/src/wallet/coincontrol.cpp index ad2fab4b7d..2087119db9 100644 --- a/src/wallet/coincontrol.cpp +++ b/src/wallet/coincontrol.cpp @@ -11,4 +11,72 @@ CCoinControl::CCoinControl() { m_avoid_partial_spends = gArgs.GetBoolArg("-avoidpartialspends", DEFAULT_AVOIDPARTIALSPENDS); } + +bool CCoinControl::HasSelected() const +{ + return !m_selected_inputs.empty(); +} + +bool CCoinControl::IsSelected(const COutPoint& output) const +{ + return m_selected_inputs.count(output) > 0; +} + +bool CCoinControl::IsExternalSelected(const COutPoint& output) const +{ + return m_external_txouts.count(output) > 0; +} + +std::optional<CTxOut> CCoinControl::GetExternalOutput(const COutPoint& outpoint) const +{ + const auto ext_it = m_external_txouts.find(outpoint); + if (ext_it == m_external_txouts.end()) { + return std::nullopt; + } + + return std::make_optional(ext_it->second); +} + +void CCoinControl::Select(const COutPoint& output) +{ + m_selected_inputs.insert(output); +} + +void CCoinControl::SelectExternal(const COutPoint& outpoint, const CTxOut& txout) +{ + m_selected_inputs.insert(outpoint); + m_external_txouts.emplace(outpoint, txout); +} + +void CCoinControl::UnSelect(const COutPoint& output) +{ + m_selected_inputs.erase(output); +} + +void CCoinControl::UnSelectAll() +{ + m_selected_inputs.clear(); +} + +std::vector<COutPoint> CCoinControl::ListSelected() const +{ + return {m_selected_inputs.begin(), m_selected_inputs.end()}; +} + +void CCoinControl::SetInputWeight(const COutPoint& outpoint, int64_t weight) +{ + m_input_weights[outpoint] = weight; +} + +bool CCoinControl::HasInputWeight(const COutPoint& outpoint) const +{ + return m_input_weights.count(outpoint) > 0; +} + +int64_t CCoinControl::GetInputWeight(const COutPoint& outpoint) const +{ + auto it = m_input_weights.find(outpoint); + assert(it != m_input_weights.end()); + return it->second; +} } // namespace wallet diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index cb6f0a1635..7ff8fee5bc 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -13,9 +13,9 @@ #include <script/signingprovider.h> #include <script/standard.h> -#include <optional> #include <algorithm> #include <map> +#include <optional> #include <set> namespace wallet { @@ -63,76 +63,62 @@ public: CCoinControl(); - bool HasSelected() const - { - return (setSelected.size() > 0); - } - - bool IsSelected(const COutPoint& output) const - { - return (setSelected.count(output) > 0); - } - - bool IsExternalSelected(const COutPoint& output) const - { - return (m_external_txouts.count(output) > 0); - } - - bool GetExternalOutput(const COutPoint& outpoint, CTxOut& txout) const - { - const auto ext_it = m_external_txouts.find(outpoint); - if (ext_it == m_external_txouts.end()) { - return false; - } - txout = ext_it->second; - return true; - } - - void Select(const COutPoint& output) - { - setSelected.insert(output); - } - - void SelectExternal(const COutPoint& outpoint, const CTxOut& txout) - { - setSelected.insert(outpoint); - m_external_txouts.emplace(outpoint, txout); - } - - void UnSelect(const COutPoint& output) - { - setSelected.erase(output); - } - - void UnSelectAll() - { - setSelected.clear(); - } - - void ListSelected(std::vector<COutPoint>& vOutpoints) const - { - vOutpoints.assign(setSelected.begin(), setSelected.end()); - } - - void SetInputWeight(const COutPoint& outpoint, int64_t weight) - { - m_input_weights[outpoint] = weight; - } - - bool HasInputWeight(const COutPoint& outpoint) const - { - return m_input_weights.count(outpoint) > 0; - } - - int64_t GetInputWeight(const COutPoint& outpoint) const - { - auto it = m_input_weights.find(outpoint); - assert(it != m_input_weights.end()); - return it->second; - } + /** + * Returns true if there are pre-selected inputs. + */ + bool HasSelected() const; + /** + * Returns true if the given output is pre-selected. + */ + bool IsSelected(const COutPoint& output) const; + /** + * Returns true if the given output is selected as an external input. + */ + bool IsExternalSelected(const COutPoint& output) const; + /** + * Returns the external output for the given outpoint if it exists. + */ + std::optional<CTxOut> GetExternalOutput(const COutPoint& outpoint) const; + /** + * Lock-in the given output for spending. + * The output will be included in the transaction even if it's not the most optimal choice. + */ + void Select(const COutPoint& output); + /** + * Lock-in the given output as an external input for spending because it is not in the wallet. + * The output will be included in the transaction even if it's not the most optimal choice. + */ + void SelectExternal(const COutPoint& outpoint, const CTxOut& txout); + /** + * Unselects the given output. + */ + void UnSelect(const COutPoint& output); + /** + * Unselects all outputs. + */ + void UnSelectAll(); + /** + * List the selected inputs. + */ + std::vector<COutPoint> ListSelected() const; + /** + * Set an input's weight. + */ + void SetInputWeight(const COutPoint& outpoint, int64_t weight); + /** + * Returns true if the input weight is set. + */ + bool HasInputWeight(const COutPoint& outpoint) const; + /** + * Returns the input weight. + */ + int64_t GetInputWeight(const COutPoint& outpoint) const; private: - std::set<COutPoint> setSelected; + //! Selected inputs (inputs that will be used, regardless of whether they're optimal or not) + std::set<COutPoint> m_selected_inputs; + //! Map of external inputs to include in the transaction + //! These are not in the wallet, so we need to track them separately std::map<COutPoint, CTxOut> m_external_txouts; //! Map of COutPoints to the maximum weight for that input std::map<COutPoint, int64_t> m_input_weights; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 832d24746f..bd74025f09 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -4,13 +4,13 @@ #include <wallet/coinselection.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <logging.h> #include <policy/feerate.h> #include <util/check.h> #include <util/moneystr.h> -#include <util/system.h> #include <numeric> #include <optional> diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 723f5bbfb3..432d7d1431 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -11,8 +11,8 @@ #include <policy/feerate.h> #include <primitives/transaction.h> #include <random.h> -#include <util/system.h> #include <util/check.h> +#include <util/insert.h> #include <util/result.h> #include <optional> diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index cd414b3d44..e2799c2d05 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -4,9 +4,9 @@ #include <wallet/crypter.h> +#include <common/system.h> #include <crypto/aes.h> #include <crypto/sha512.h> -#include <util/system.h> #include <vector> diff --git a/src/wallet/db.h b/src/wallet/db.h index 6834ba6963..9d684225c3 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -113,6 +113,7 @@ public: virtual bool ErasePrefix(Span<const std::byte> prefix) = 0; virtual std::unique_ptr<DatabaseCursor> GetNewCursor() = 0; + virtual std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) = 0; virtual bool TxnBegin() = 0; virtual bool TxnCommit() = 0; virtual bool TxnAbort() = 0; @@ -174,51 +175,6 @@ public: virtual std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) = 0; }; -class DummyCursor : public DatabaseCursor -{ - Status Next(DataStream& key, DataStream& value) override { return Status::FAIL; } -}; - -/** RAII class that provides access to a DummyDatabase. Never fails. */ -class DummyBatch : public DatabaseBatch -{ -private: - bool ReadKey(DataStream&& key, DataStream& value) override { return true; } - bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return true; } - bool EraseKey(DataStream&& key) override { return true; } - bool HasKey(DataStream&& key) override { return true; } - bool ErasePrefix(Span<const std::byte> prefix) override { return true; } - -public: - void Flush() override {} - void Close() override {} - - std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<DummyCursor>(); } - bool TxnBegin() override { return true; } - bool TxnCommit() override { return true; } - bool TxnAbort() override { return true; } -}; - -/** A dummy WalletDatabase that does nothing and never fails. Only used by unit tests. - **/ -class DummyDatabase : public WalletDatabase -{ -public: - void Open() override {}; - void AddRef() override {} - void RemoveRef() override {} - bool Rewrite(const char* pszSkip=nullptr) override { return true; } - bool Backup(const std::string& strDest) const override { return true; } - void Close() override {} - void Flush() override {} - bool PeriodicFlush() override { return true; } - void IncrementUpdateCounter() override { ++nUpdateCounter; } - void ReloadDbEnv() override {} - std::string Filename() override { return "dummy"; } - std::string Format() override { return "dummy"; } - std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override { return std::make_unique<DummyBatch>(); } -}; - enum class DatabaseFormat { BERKELEY, SQLITE, diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 865af0bb23..44c93eed7c 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -255,11 +255,7 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: std::vector<unsigned char> k = ParseHex(key); std::vector<unsigned char> v = ParseHex(value); - - DataStream ss_key{k}; - DataStream ss_value{v}; - - if (!batch->Write(ss_key, ss_value)) { + if (!batch->Write(Span{k}, Span{v})) { error = strprintf(_("Error: Unable to write record to new wallet")); ret = false; break; diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index 079e3baa4e..6d026d02c1 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -4,8 +4,8 @@ #include <chainparams.h> #include <common/args.h> +#include <common/system.h> #include <external_signer.h> -#include <util/system.h> #include <wallet/external_signer_scriptpubkeyman.h> #include <iostream> @@ -45,7 +45,7 @@ ExternalSigner ExternalSignerScriptPubKeyMan::GetExternalSigner() { const std::string command = gArgs.GetArg("-signer", ""); if (command == "") throw std::runtime_error(std::string(__func__) + ": restart bitcoind with -signer=<cmd>"); std::vector<ExternalSigner> signers; - ExternalSigner::Enumerate(command, signers, Params().NetworkIDString()); + ExternalSigner::Enumerate(command, signers, Params().GetChainTypeString()); if (signers.empty()) throw std::runtime_error(std::string(__func__) + ": No external signers found"); // TODO: add fingerprint argument instead of failing in case of multiple signers. if (signers.size() > 1) throw std::runtime_error(std::string(__func__) + ": More than one external signer found. Please connect only one at a time."); diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index b6b1fa1d3e..0e1f1f9847 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <consensus/validation.h> #include <interfaces/chain.h> #include <policy/fees.h> #include <policy/policy.h> #include <util/moneystr.h> #include <util/rbf.h> -#include <util/system.h> #include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/feebumper.h> diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 2560dda87c..4cdfadbee2 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -62,7 +62,7 @@ bool VerifyWallets(WalletContext& context) options.require_existing = true; options.verify = false; if (MakeWalletDatabase("", options, status, error_string)) { - util::SettingsValue wallets(util::SettingsValue::VARR); + common::SettingsValue wallets(common::SettingsValue::VARR); wallets.push_back(""); // Default wallet name is "" // Pass write=false because no need to write file and probably // better not to. If unnamed wallet needs to be added next startup diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index a5be0739a9..0bd6a9670c 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -608,7 +608,9 @@ RPCHelpMan getaddressinfo() if (const std::unique_ptr<CKeyMetadata> meta = spk_man->GetMetadata(dest)) { ret.pushKV("timestamp", meta->nCreateTime); if (meta->has_key_origin) { - ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); + // In legacy wallets hdkeypath has always used an apostrophe for + // hardened derivation. Perhaps some external tool depends on that. + ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path, /*apostrophe=*/!desc_spk_man)); ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint)); } diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 553bbfb62f..b93419292e 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -805,7 +805,7 @@ RPCHelpMan dumpwallet() } else { file << "change=1"; } - file << strprintf(" # addr=%s%s\n", strAddr, (metadata.has_key_origin ? " hdkeypath="+WriteHDKeypath(metadata.key_origin.path) : "")); + file << strprintf(" # addr=%s%s\n", strAddr, (metadata.has_key_origin ? " hdkeypath="+WriteHDKeypath(metadata.key_origin.path, /*apostrophe=*/true) : "")); } } file << "\n"; @@ -1298,7 +1298,7 @@ RPCHelpMan importmulti() }, }, RPCArgOptions{.oneline_description="\"requests\""}}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions after all imports."}, }, diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 750ef69f6e..22f0f0b83c 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -320,7 +320,7 @@ RPCHelpMan lockunspent() }); const uint256 txid(ParseHashO(o, "txid")); - const int nOutput = find_value(o, "vout").getInt<int>(); + const int nOutput = o.find_value("vout").getInt<int>(); if (nOutput < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout cannot be negative"); } @@ -513,7 +513,7 @@ RPCHelpMan listunspent() }, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include outputs that are not safe to spend\n" "See description of \"safe\" attribute below."}, - {"query_options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "JSON with query options", + {"query_options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"minimumAmount", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(0)}, "Minimum value of each UTXO in " + CURRENCY_UNIT + ""}, {"maximumAmount", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"unlimited"}, "Maximum value of each UTXO in " + CURRENCY_UNIT + ""}, diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 88ee6e96b0..feee643f26 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -453,9 +453,9 @@ RPCHelpMan settxfee() static std::vector<RPCArg> FundTxDoc(bool solving_data = true) { std::vector<RPCArg> args = { - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, + {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks", RPCArgOptions{.also_positional = true}}, {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, "The fee estimate mode, must be one of (case insensitive):\n" - "\"" + FeeModes("\"\n\"") + "\""}, + "\"" + FeeModes("\"\n\"") + "\"", RPCArgOptions{.also_positional = true}}, { "replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125-replaceable.\n" "Allows this transaction to be replaced by a transaction with higher fees" @@ -673,7 +673,7 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, for (const UniValue& input : options["input_weights"].get_array().getValues()) { uint256 txid = ParseHashO(input, "txid"); - const UniValue& vout_v = find_value(input, "vout"); + const UniValue& vout_v = input.find_value("vout"); if (!vout_v.isNum()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, missing vout key"); } @@ -682,7 +682,7 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout cannot be negative"); } - const UniValue& weight_v = find_value(input, "weight"); + const UniValue& weight_v = input.find_value("weight"); if (!weight_v.isNum()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, missing weight key"); } @@ -758,7 +758,7 @@ RPCHelpMan fundrawtransaction() "Only pay-to-pubkey, multisig, and P2SH versions thereof are currently supported for watch-only\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "For backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{true}, "For a transaction with existing inputs, automatically include more if they are not enough."}, @@ -997,7 +997,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) "* WARNING: before version 0.21, fee_rate was in " + CURRENCY_UNIT + "/kvB. As of 0.21, fee_rate is in " + CURRENCY_ATOM + "/vB. *\n", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The txid to be bumped"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks\n"}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, @@ -1187,7 +1187,7 @@ RPCHelpMan send() {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, "The fee estimate mode, must be one of (case insensitive):\n" "\"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::DefaultHint{"false when \"inputs\" are specified, true otherwise"},"Automatically include coins from the wallet to cover the target amount.\n"}, @@ -1200,7 +1200,7 @@ RPCHelpMan send() {"change_address", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"change_position", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", \"bech32\" and \"bech32m\"."}, - {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, + {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB.", RPCArgOptions{.also_positional = true}}, {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, @@ -1302,11 +1302,11 @@ RPCHelpMan sendall() "\"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, { - "options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + "options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_to_wallet", RPCArg::Type::BOOL, RPCArg::Default{true}, "When false, returns the serialized transaction without broadcasting or adding it to the wallet"}, - {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, + {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB.", RPCArgOptions{.also_positional = true}}, {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch-only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, @@ -1635,7 +1635,7 @@ RPCHelpMan walletcreatefundedpsbt() OutputsDoc(), RPCArgOptions{.skip_type_check = true}}, {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::DefaultHint{"false when \"inputs\" are specified, true otherwise"}, "Automatically include coins from the wallet to cover the target amount.\n"}, diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 4ff44b84b0..06ec7db1bc 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -6,7 +6,7 @@ #include <common/url.h> #include <rpc/util.h> -#include <util/system.h> +#include <util/any.h> #include <util/translation.h> #include <wallet/context.h> #include <wallet/wallet.h> diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index d71359508d..340e4115af 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -123,7 +123,7 @@ static RPCHelpMan getwalletinfo() obj.pushKV("avoid_reuse", pwallet->IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE)); if (pwallet->IsScanning()) { UniValue scanning(UniValue::VOBJ); - scanning.pushKV("duration", pwallet->ScanningDuration() / 1000); + scanning.pushKV("duration", Ticks<std::chrono::seconds>(pwallet->ScanningDuration())); scanning.pushKV("progress", pwallet->ScanningProgress()); obj.pushKV("scanning", scanning); } else { @@ -647,7 +647,7 @@ RPCHelpMan simulaterawtransaction() {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, }, - {"options", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED, "Options", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"include_watchonly", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Whether to include watch-only addresses (see RPC importaddress)"}, }, diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 06c8c8bb37..e303310273 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -23,6 +23,52 @@ static bool KeyFilter(const std::string& type) return WalletBatch::IsKeyType(type) || type == DBKeys::HDCHAIN; } +class DummyCursor : public DatabaseCursor +{ + Status Next(DataStream& key, DataStream& value) override { return Status::FAIL; } +}; + +/** RAII class that provides access to a DummyDatabase. Never fails. */ +class DummyBatch : public DatabaseBatch +{ +private: + bool ReadKey(DataStream&& key, DataStream& value) override { return true; } + bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite=true) override { return true; } + bool EraseKey(DataStream&& key) override { return true; } + bool HasKey(DataStream&& key) override { return true; } + bool ErasePrefix(Span<const std::byte> prefix) override { return true; } + +public: + void Flush() override {} + void Close() override {} + + std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<DummyCursor>(); } + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override { return GetNewCursor(); } + bool TxnBegin() override { return true; } + bool TxnCommit() override { return true; } + bool TxnAbort() override { return true; } +}; + +/** A dummy WalletDatabase that does nothing and never fails. Only used by salvage. + **/ +class DummyDatabase : public WalletDatabase +{ +public: + void Open() override {}; + void AddRef() override {} + void RemoveRef() override {} + bool Rewrite(const char* pszSkip=nullptr) override { return true; } + bool Backup(const std::string& strDest) const override { return true; } + void Close() override {} + void Flush() override {} + bool PeriodicFlush() override { return true; } + void IncrementUpdateCounter() override { ++nUpdateCounter; } + void ReloadDbEnv() override {} + std::string Filename() override { return "dummy"; } + std::string Format() override { return "dummy"; } + std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override { return std::make_unique<DummyBatch>(); } +}; + bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings) { DatabaseOptions options; @@ -135,7 +181,7 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil } DbTxn* ptxn = env->TxnBegin(); - CWallet dummyWallet(nullptr, "", CreateDummyWalletDatabase()); + CWallet dummyWallet(nullptr, "", std::make_unique<DummyDatabase>()); for (KeyValPair& row : salvagedData) { /* Filter for only private key type KV pairs to be added to the salvaged wallet */ diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 8c34c45e49..796b7f11c5 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -710,6 +710,8 @@ void LegacyScriptPubKeyMan::UpdateTimeFirstKey(int64_t nCreateTime) } else if (!nTimeFirstKey || nCreateTime < nTimeFirstKey) { nTimeFirstKey = nCreateTime; } + + NotifyFirstKeyTimeChanged(this, nTimeFirstKey); } bool LegacyScriptPubKeyMan::LoadKey(const CKey& key, const CPubKey &pubkey) @@ -1555,7 +1557,7 @@ bool LegacyScriptPubKeyMan::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; mapKeyMetadata[pubkey.GetID()].has_key_origin = true; - mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path); + mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path, /*apostrophe=*/true); return batch.WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true); } @@ -1821,7 +1823,7 @@ std::optional<MigrationData> LegacyScriptPubKeyMan::MigrateToDescriptor() // Make the combo descriptor std::string xpub = EncodeExtPubKey(master_key.Neuter()); - std::string desc_str = "combo(" + xpub + "/0'/" + ToString(i) + "'/*')"; + std::string desc_str = "combo(" + xpub + "/0h/" + ToString(i) + "h/*h)"; FlatSigningProvider keys; std::string error; std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, error, false); @@ -2264,20 +2266,20 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ std::string desc_suffix = "/*)"; switch (addr_type) { case OutputType::LEGACY: { - desc_prefix = "pkh(" + xpub + "/44'"; + desc_prefix = "pkh(" + xpub + "/44h"; break; } case OutputType::P2SH_SEGWIT: { - desc_prefix = "sh(wpkh(" + xpub + "/49'"; + desc_prefix = "sh(wpkh(" + xpub + "/49h"; desc_suffix += ")"; break; } case OutputType::BECH32: { - desc_prefix = "wpkh(" + xpub + "/84'"; + desc_prefix = "wpkh(" + xpub + "/84h"; break; } case OutputType::BECH32M: { - desc_prefix = "tr(" + xpub + "/86'"; + desc_prefix = "tr(" + xpub + "/86h"; break; } case OutputType::UNKNOWN: { @@ -2290,13 +2292,13 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ // Mainnet derives at 0', testnet and regtest derive at 1' if (Params().IsTestChain()) { - desc_prefix += "/1'"; + desc_prefix += "/1h"; } else { - desc_prefix += "/0'"; + desc_prefix += "/0h"; } std::string internal_path = internal ? "/1" : "/0"; - std::string desc_str = desc_prefix + "/0'" + internal_path + desc_suffix; + std::string desc_str = desc_prefix + "/0h" + internal_path + desc_suffix; // Make the descriptor FlatSigningProvider keys; @@ -2736,6 +2738,8 @@ void DescriptorScriptPubKeyMan::UpdateWalletDescriptor(WalletDescriptor& descrip m_map_script_pub_keys.clear(); m_max_cached_index = -1; m_wallet_descriptor = descriptor; + + NotifyFirstKeyTimeChanged(this, m_wallet_descriptor.creation_time); } bool DescriptorScriptPubKeyMan::CanUpdateToWalletDescriptor(const WalletDescriptor& descriptor, std::string& error) diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 22b67c88e9..bf35c776ae 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_SCRIPTPUBKEYMAN_H #define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H +#include <logging.h> #include <psbt.h> #include <script/descriptor.h> #include <script/signingprovider.h> @@ -27,6 +28,8 @@ enum class OutputType; struct bilingual_str; namespace wallet { +struct MigrationData; + // Wallet storage things that ScriptPubKeyMans need in order to be able to store things to the wallet database. // It provides access to things that are part of the entire wallet and not specific to a ScriptPubKeyMan such as // wallet flags, wallet version, encryption keys, encryption status, and the database itself. This allows a @@ -256,6 +259,9 @@ public: /** Keypool has new keys */ boost::signals2::signal<void ()> NotifyCanGetAddressesChanged; + + /** Birth time changed */ + boost::signals2::signal<void (const ScriptPubKeyMan* spkm, int64_t new_birth_time)> NotifyFirstKeyTimeChanged; }; /** OutputTypes supported by the LegacyScriptPubKeyMan */ @@ -658,6 +664,18 @@ public: void UpgradeDescriptorCache(); }; + +/** struct containing information needed for migrating legacy wallets to descriptor wallets */ +struct MigrationData +{ + CExtKey master_key; + std::vector<std::pair<std::string, int64_t>> watch_descs; + std::vector<std::pair<std::string, int64_t>> solvable_descs; + std::vector<std::unique_ptr<DescriptorScriptPubKeyMan>> desc_spkms; + std::shared_ptr<CWallet> watchonly_wallet{nullptr}; + std::shared_ptr<CWallet> solvable_wallet{nullptr}; +}; + } // namespace wallet #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 748f40dce8..99c6582f9c 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -4,6 +4,7 @@ #include <algorithm> #include <common/args.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/validation.h> #include <interfaces/chain.h> @@ -15,7 +16,6 @@ #include <util/fees.h> #include <util/moneystr.h> #include <util/rbf.h> -#include <util/system.h> #include <util/trace.h> #include <util/translation.h> #include <wallet/coincontrol.h> @@ -52,9 +52,7 @@ int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control) { CMutableTransaction txNew(tx); - if (!wallet->DummySignTx(txNew, txouts, coin_control)) { - return TxSize{-1, -1}; - } + if (!wallet->DummySignTx(txNew, txouts, coin_control)) return TxSize{-1, -1}; CTransaction ctx(txNew); int64_t vsize = GetVirtualTransactionSize(ctx); int64_t weight = GetTransactionWeight(ctx); @@ -72,11 +70,9 @@ TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *walle assert(input.prevout.n < mi->second.tx->vout.size()); txouts.emplace_back(mi->second.tx->vout.at(input.prevout.n)); } else if (coin_control) { - CTxOut txout; - if (!coin_control->GetExternalOutput(input.prevout, txout)) { - return TxSize{-1, -1}; - } - txouts.emplace_back(txout); + const auto& txout{coin_control->GetExternalOutput(input.prevout)}; + if (!txout) return TxSize{-1, -1}; + txouts.emplace_back(*txout); } else { return TxSize{-1, -1}; } @@ -163,10 +159,8 @@ util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const const CoinSelectionParams& coin_selection_params) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { PreSelectedInputs result; - std::vector<COutPoint> vPresetInputs; - coin_control.ListSelected(vPresetInputs); const bool can_grind_r = wallet.CanGrindR(); - for (const COutPoint& outpoint : vPresetInputs) { + for (const COutPoint& outpoint : coin_control.ListSelected()) { int input_bytes = -1; CTxOut txout; if (auto ptr_wtx = wallet.GetWalletTx(outpoint.hash)) { @@ -178,9 +172,12 @@ util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const input_bytes = CalculateMaximumSignedInputSize(txout, &wallet, &coin_control); } else { // The input is external. We did not find the tx in mapWallet. - if (!coin_control.GetExternalOutput(outpoint, txout)) { + const auto out{coin_control.GetExternalOutput(outpoint)}; + if (!out) { return util::Error{strprintf(_("Not found pre-selected input %s"), outpoint.ToString())}; } + + txout = *out; } if (input_bytes == -1) { diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 77e8a4e9c1..8d7fe97bb1 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -9,6 +9,7 @@ #include <logging.h> #include <sync.h> #include <util/fs_helpers.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/translation.h> #include <wallet/db.h> @@ -34,12 +35,31 @@ static void ErrorLogCallback(void* arg, int code, const char* msg) LogPrintf("SQLite Error. Code: %d. Message: %s\n", code, msg); } +static int TraceSqlCallback(unsigned code, void* context, void* param1, void* param2) +{ + auto* db = static_cast<SQLiteDatabase*>(context); + if (code == SQLITE_TRACE_STMT) { + auto* stmt = static_cast<sqlite3_stmt*>(param1); + // To be conservative and avoid leaking potentially secret information + // in the log file, only expand statements that query the database, not + // statements that update the database. + char* expanded{sqlite3_stmt_readonly(stmt) ? sqlite3_expanded_sql(stmt) : nullptr}; + LogPrintf("[%s] SQLite Statement: %s\n", db->Filename(), expanded ? expanded : sqlite3_sql(stmt)); + if (expanded) sqlite3_free(expanded); + } + return SQLITE_OK; +} + static bool BindBlobToStatement(sqlite3_stmt* stmt, int index, Span<const std::byte> blob, const std::string& description) { - int res = sqlite3_bind_blob(stmt, index, blob.data(), blob.size(), SQLITE_STATIC); + // Pass a pointer to the empty string "" below instead of passing the + // blob.data() pointer if the blob.data() pointer is null. Passing a null + // data pointer to bind_blob would cause sqlite to bind the SQL NULL value + // instead of the empty blob value X'', which would mess up SQL comparisons. + int res = sqlite3_bind_blob(stmt, index, blob.data() ? static_cast<const void*>(blob.data()) : "", blob.size(), SQLITE_STATIC); if (res != SQLITE_OK) { LogPrintf("Unable to bind %s to statement: %s\n", description, sqlite3_errstr(res)); sqlite3_clear_bindings(stmt); @@ -235,6 +255,13 @@ void SQLiteDatabase::Open() if (ret != SQLITE_OK) { throw std::runtime_error(strprintf("SQLiteDatabase: Failed to enable extended result codes: %s\n", sqlite3_errstr(ret))); } + // Trace SQL statements if tracing is enabled with -debug=walletdb -loglevel=walletdb:trace + if (LogAcceptCategory(BCLog::WALLETDB, BCLog::Level::Trace)) { + ret = sqlite3_trace_v2(m_db, SQLITE_TRACE_STMT, TraceSqlCallback, this); + if (ret != SQLITE_OK) { + LogPrintf("Failed to enable SQL tracing for %s\n", Filename()); + } + } } if (sqlite3_db_readonly(m_db, "main") != 0) { @@ -409,6 +436,7 @@ bool SQLiteBatch::ReadKey(DataStream&& key, DataStream& value) // Leftmost column in result is index 0 const std::byte* data{AsBytePtr(sqlite3_column_blob(m_read_stmt, 0))}; size_t data_size(sqlite3_column_bytes(m_read_stmt, 0)); + value.clear(); value.write({data, data_size}); sqlite3_clear_bindings(m_read_stmt); @@ -495,6 +523,9 @@ DatabaseCursor::Status SQLiteCursor::Next(DataStream& key, DataStream& value) return Status::FAIL; } + key.clear(); + value.clear(); + // Leftmost column in result is index 0 const std::byte* key_data{AsBytePtr(sqlite3_column_blob(m_cursor_stmt, 0))}; size_t key_data_size(sqlite3_column_bytes(m_cursor_stmt, 0)); @@ -507,6 +538,7 @@ DatabaseCursor::Status SQLiteCursor::Next(DataStream& key, DataStream& value) SQLiteCursor::~SQLiteCursor() { + sqlite3_clear_bindings(m_cursor_stmt); sqlite3_reset(m_cursor_stmt); int res = sqlite3_finalize(m_cursor_stmt); if (res != SQLITE_OK) { @@ -530,6 +562,48 @@ std::unique_ptr<DatabaseCursor> SQLiteBatch::GetNewCursor() return cursor; } +std::unique_ptr<DatabaseCursor> SQLiteBatch::GetNewPrefixCursor(Span<const std::byte> prefix) +{ + if (!m_database.m_db) return nullptr; + + // To get just the records we want, the SQL statement does a comparison of the binary data + // where the data must be greater than or equal to the prefix, and less than + // the prefix incremented by one (when interpreted as an integer) + std::vector<std::byte> start_range(prefix.begin(), prefix.end()); + std::vector<std::byte> end_range(prefix.begin(), prefix.end()); + auto it = end_range.rbegin(); + for (; it != end_range.rend(); ++it) { + if (*it == std::byte(std::numeric_limits<unsigned char>::max())) { + *it = std::byte(0); + continue; + } + *it = std::byte(std::to_integer<unsigned char>(*it) + 1); + break; + } + if (it == end_range.rend()) { + // If the prefix is all 0xff bytes, clear end_range as we won't need it + end_range.clear(); + } + + auto cursor = std::make_unique<SQLiteCursor>(start_range, end_range); + if (!cursor) return nullptr; + + const char* stmt_text = end_range.empty() ? "SELECT key, value FROM main WHERE key >= ?" : + "SELECT key, value FROM main WHERE key >= ? AND key < ?"; + int res = sqlite3_prepare_v2(m_database.m_db, stmt_text, -1, &cursor->m_cursor_stmt, nullptr); + if (res != SQLITE_OK) { + throw std::runtime_error(strprintf( + "SQLiteDatabase: Failed to setup cursor SQL statement: %s\n", sqlite3_errstr(res))); + } + + if (!BindBlobToStatement(cursor->m_cursor_stmt, 1, cursor->m_prefix_range_start, "prefix_start")) return nullptr; + if (!end_range.empty()) { + if (!BindBlobToStatement(cursor->m_cursor_stmt, 2, cursor->m_prefix_range_end, "prefix_end")) return nullptr; + } + + return cursor; +} + bool SQLiteBatch::TxnBegin() { if (!m_database.m_db || sqlite3_get_autocommit(m_database.m_db) == 0) return false; diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index d9de40569b..0378bbb8d6 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -15,12 +15,21 @@ struct bilingual_str; namespace wallet { class SQLiteDatabase; +/** RAII class that provides a database cursor */ class SQLiteCursor : public DatabaseCursor { public: sqlite3_stmt* m_cursor_stmt{nullptr}; + // Copies of the prefix things for the prefix cursor. + // Prevents SQLite from accessing temp variables for the prefix things. + std::vector<std::byte> m_prefix_range_start; + std::vector<std::byte> m_prefix_range_end; explicit SQLiteCursor() {} + explicit SQLiteCursor(std::vector<std::byte> start_range, std::vector<std::byte> end_range) + : m_prefix_range_start(std::move(start_range)), + m_prefix_range_end(std::move(end_range)) + {} ~SQLiteCursor() override; Status Next(DataStream& key, DataStream& value) override; @@ -57,6 +66,7 @@ public: void Close() override; std::unique_ptr<DatabaseCursor> GetNewCursor() override; + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override; bool TxnBegin() override; bool TxnCommit() override; bool TxnAbort() override; diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 7f66179517..b1d67c1432 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -12,6 +12,7 @@ #include <wallet/coincontrol.h> #include <wallet/coinselection.h> #include <wallet/spend.h> +#include <wallet/test/util.h> #include <wallet/test/wallet_test_fixture.h> #include <wallet/wallet.h> @@ -176,7 +177,7 @@ inline std::vector<OutputGroup>& KnapsackGroupOutputs(const CoinsResult& availab static std::unique_ptr<CWallet> NewWallet(const node::NodeContext& m_node, const std::string& wallet_name = "") { - std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), wallet_name, CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), wallet_name, CreateMockableWalletDatabase()); BOOST_CHECK(wallet->LoadWallet() == DBErrors::LOAD_OK); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -431,7 +432,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) CAmount selection_target = 16 * CENT; const auto& no_res = SelectCoinsBnB(GroupCoins(available_coins.All(), /*subtract_fee_outputs*/true), selection_target, /*cost_of_change=*/0, MAX_STANDARD_TX_WEIGHT); - BOOST_ASSERT(!no_res); + BOOST_REQUIRE(!no_res); BOOST_CHECK(util::ErrorString(no_res).original.find("The inputs size exceeds the maximum weight") != std::string::npos); // Now add same coin value with a good size and check that it gets selected diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 7761308bbc..4cda35ed8d 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -6,13 +6,56 @@ #include <test/util/setup_common.h> #include <util/fs.h> +#include <util/translation.h> +#ifdef USE_BDB #include <wallet/bdb.h> +#endif +#ifdef USE_SQLITE +#include <wallet/sqlite.h> +#endif +#include <wallet/test/util.h> +#include <wallet/walletutil.h> // for WALLET_FLAG_DESCRIPTORS #include <fstream> #include <memory> #include <string> +inline std::ostream& operator<<(std::ostream& os, const std::pair<const SerializeData, SerializeData>& kv) +{ + Span key{kv.first}, value{kv.second}; + os << "(\"" << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\", \"" + << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\")"; + return os; +} + namespace wallet { + +static Span<const std::byte> StringBytes(std::string_view str) +{ + return AsBytes<const char>({str.data(), str.size()}); +} + +static SerializeData StringData(std::string_view str) +{ + auto bytes = StringBytes(str); + return SerializeData{bytes.begin(), bytes.end()}; +} + +static void CheckPrefix(DatabaseBatch& batch, Span<const std::byte> prefix, MockableData expected) +{ + std::unique_ptr<DatabaseCursor> cursor = batch.GetNewPrefixCursor(prefix); + MockableData actual; + while (true) { + DataStream key, value; + DatabaseCursor::Status status = cursor->Next(key, value); + if (status == DatabaseCursor::Status::DONE) break; + BOOST_CHECK(status == DatabaseCursor::Status::MORE); + BOOST_CHECK( + actual.emplace(SerializeData(key.begin(), key.end()), SerializeData(value.begin(), value.end())).second); + } + BOOST_CHECK_EQUAL_COLLECTIONS(actual.begin(), actual.end(), expected.begin(), expected.end()); +} + BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, fs::path& database_filename) @@ -78,5 +121,90 @@ BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_free_instance) BOOST_CHECK(env_2_a == env_2_b); } +static std::vector<std::unique_ptr<WalletDatabase>> TestDatabases(const fs::path& path_root) +{ + std::vector<std::unique_ptr<WalletDatabase>> dbs; + DatabaseOptions options; + DatabaseStatus status; + bilingual_str error; +#ifdef USE_BDB + dbs.emplace_back(MakeBerkeleyDatabase(path_root / "bdb", options, status, error)); +#endif +#ifdef USE_SQLITE + dbs.emplace_back(MakeSQLiteDatabase(path_root / "sqlite", options, status, error)); +#endif + dbs.emplace_back(CreateMockableWalletDatabase()); + return dbs; +} + +BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test) +{ + // Test each supported db + for (const auto& database : TestDatabases(m_path_root)) { + BOOST_ASSERT(database); + + std::vector<std::string> prefixes = {"", "FIRST", "SECOND", "P\xfe\xff", "P\xff\x01", "\xff\xff"}; + + // Write elements to it + std::unique_ptr<DatabaseBatch> handler = database->MakeBatch(); + for (unsigned int i = 0; i < 10; i++) { + for (const auto& prefix : prefixes) { + BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i)); + } + } + + // Now read all the items by prefix and verify that each element gets parsed correctly + for (const auto& prefix : prefixes) { + DataStream s_prefix; + s_prefix << prefix; + std::unique_ptr<DatabaseCursor> cursor = handler->GetNewPrefixCursor(s_prefix); + DataStream key; + DataStream value; + for (int i = 0; i < 10; i++) { + DatabaseCursor::Status status = cursor->Next(key, value); + BOOST_ASSERT(status == DatabaseCursor::Status::MORE); + + std::string key_back; + unsigned int i_back; + key >> key_back >> i_back; + BOOST_CHECK_EQUAL(key_back, prefix); + + unsigned int value_back; + value >> value_back; + BOOST_CHECK_EQUAL(value_back, i_back); + } + + // Let's now read it once more, it should return DONE + BOOST_CHECK(cursor->Next(key, value) == DatabaseCursor::Status::DONE); + } + } +} + +// Lower level DatabaseBase::GetNewPrefixCursor test, to cover cases that aren't +// covered in the higher level test above. The higher level test uses +// serialized strings which are prefixed with string length, so it doesn't test +// truly empty prefixes or prefixes that begin with \xff +BOOST_AUTO_TEST_CASE(db_cursor_prefix_byte_test) +{ + const MockableData::value_type + e{StringData(""), StringData("e")}, + p{StringData("prefix"), StringData("p")}, + ps{StringData("prefixsuffix"), StringData("ps")}, + f{StringData("\xff"), StringData("f")}, + fs{StringData("\xffsuffix"), StringData("fs")}, + ff{StringData("\xff\xff"), StringData("ff")}, + ffs{StringData("\xff\xffsuffix"), StringData("ffs")}; + for (const auto& database : TestDatabases(m_path_root)) { + std::unique_ptr<DatabaseBatch> batch = database->MakeBatch(); + for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) { + batch->Write(MakeUCharSpan(k), MakeUCharSpan(v)); + } + CheckPrefix(*batch, StringBytes(""), {e, p, ps, f, fs, ff, ffs}); + CheckPrefix(*batch, StringBytes("prefix"), {p, ps}); + CheckPrefix(*batch, StringBytes("\xff"), {f, fs, ff, ffs}); + CheckPrefix(*batch, StringBytes("\xff\xff"), {ff, ffs}); + } +} + BOOST_AUTO_TEST_SUITE_END() } // namespace wallet diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index de381a5ec9..f4b69f7403 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -141,6 +141,10 @@ FUZZ_TARGET_INIT(wallet_notifications, initialize_setup) info.prev_hash = &block.hashPrevBlock; info.height = chain.size(); info.data = █ + // Ensure that no blocks are skipped by the wallet by setting the chain's accumulated + // time to the maximum value. This ensures that the wallet's birth time is always + // earlier than this maximum time. + info.chain_time_max = std::numeric_limits<unsigned int>::max(); a.wallet->blockConnected(info); b.wallet->blockConnected(info); // Store the coins for the next block diff --git a/src/wallet/test/group_outputs_tests.cpp b/src/wallet/test/group_outputs_tests.cpp index 283e87989c..e6b25cc216 100644 --- a/src/wallet/test/group_outputs_tests.cpp +++ b/src/wallet/test/group_outputs_tests.cpp @@ -6,6 +6,7 @@ #include <wallet/coinselection.h> #include <wallet/spend.h> +#include <wallet/test/util.h> #include <wallet/wallet.h> #include <boost/test/unit_test.hpp> @@ -17,7 +18,7 @@ static int nextLockTime = 0; static std::shared_ptr<CWallet> NewWallet(const node::NodeContext& m_node) { - std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index 0adc63876c..5bdf36ec19 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -4,6 +4,7 @@ #include <common/args.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/check.h> #include <util/fs.h> @@ -13,7 +14,7 @@ #include <wallet/test/init_test_fixture.h> namespace wallet { -InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) +InitWalletDirTestingSetup::InitWalletDirTestingSetup(const ChainType chainType) : BasicTestingSetup(chainType) { m_wallet_loader = MakeWalletLoader(*m_node.chain, m_args); diff --git a/src/wallet/test/init_test_fixture.h b/src/wallet/test/init_test_fixture.h index df5819fd1d..ac7bb8997c 100644 --- a/src/wallet/test/init_test_fixture.h +++ b/src/wallet/test/init_test_fixture.h @@ -9,11 +9,12 @@ #include <interfaces/wallet.h> #include <node/context.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> namespace wallet { struct InitWalletDirTestingSetup: public BasicTestingSetup { - explicit InitWalletDirTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + explicit InitWalletDirTestingSetup(const ChainType chain_type = ChainType::MAIN); ~InitWalletDirTestingSetup(); void SetWalletDir(const fs::path& walletdir_path); diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index f6f2d423e4..fd0718fbb9 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -10,6 +10,7 @@ #include <test/util/setup_common.h> #include <wallet/types.h> #include <wallet/wallet.h> +#include <wallet/test/util.h> #include <boost/test/unit_test.hpp> @@ -55,7 +56,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); @@ -74,7 +75,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "pk(" + EncodeSecret(keys[0]) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -86,7 +87,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK uncompressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); @@ -105,7 +106,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK uncompressed - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "pk(" + EncodeSecret(uncompressedKey) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -117,7 +118,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH compressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); @@ -136,7 +137,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH compressed - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "pkh(" + EncodeSecret(keys[0]) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -148,7 +149,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH uncompressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); @@ -167,7 +168,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH uncompressed - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "pkh(" + EncodeSecret(uncompressedKey) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -179,7 +180,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -206,7 +207,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "sh(pkh(" + EncodeSecret(keys[0]) + "))"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -219,7 +220,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2SH (invalid) - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -238,7 +239,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2SH (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "sh(sh(" + EncodeSecret(keys[0]) + "))"; auto spk_manager = CreateDescriptor(keystore, desc_str, false); @@ -247,7 +248,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2WSH (invalid) - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -266,7 +267,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2WSH (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wsh(sh(" + EncodeSecret(keys[0]) + "))"; auto spk_manager = CreateDescriptor(keystore, desc_str, false); @@ -275,7 +276,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH inside P2WSH (invalid) - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -292,7 +293,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH inside P2WSH (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wsh(wpkh(" + EncodeSecret(keys[0]) + "))"; auto spk_manager = CreateDescriptor(keystore, desc_str, false); @@ -301,7 +302,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2WSH inside P2WSH (invalid) - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -320,7 +321,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2WSH inside P2WSH (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wsh(wsh(" + EncodeSecret(keys[0]) + "))"; auto spk_manager = CreateDescriptor(keystore, desc_str, false); @@ -329,7 +330,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH compressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -345,7 +346,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH compressed - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wpkh(" + EncodeSecret(keys[0]) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -357,7 +358,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH uncompressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -378,7 +379,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH uncompressed (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wpkh(" + EncodeSecret(uncompressedKey) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, false); @@ -387,7 +388,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // scriptPubKey multisig - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -422,7 +423,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // scriptPubKey multisig - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "multi(2, " + EncodeSecret(uncompressedKey) + ", " + EncodeSecret(keys[1]) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -434,7 +435,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH multisig - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -457,7 +458,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH multisig - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "sh(multi(2, " + EncodeSecret(uncompressedKey) + ", " + EncodeSecret(keys[1]) + "))"; @@ -471,7 +472,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with compressed keys - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -500,7 +501,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with compressed keys - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wsh(multi(2, " + EncodeSecret(keys[0]) + ", " + EncodeSecret(keys[1]) + "))"; @@ -514,7 +515,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with uncompressed key - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -543,7 +544,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with uncompressed key (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wsh(multi(2, " + EncodeSecret(uncompressedKey) + ", " + EncodeSecret(keys[1]) + "))"; @@ -553,7 +554,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig wrapped in P2SH - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -583,7 +584,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig wrapped in P2SH - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "sh(wsh(multi(2, " + EncodeSecret(keys[0]) + ", " + EncodeSecret(keys[1]) + ")))"; @@ -598,7 +599,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Combo - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "combo(" + EncodeSecret(keys[0]) + ")"; @@ -642,7 +643,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Taproot - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "tr(" + EncodeSecret(keys[0]) + ")"; @@ -660,7 +661,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // OP_RETURN { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -675,7 +676,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unspendable { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -690,7 +691,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unknown { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -705,7 +706,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Nonstandard { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); diff --git a/src/wallet/test/scriptpubkeyman_tests.cpp b/src/wallet/test/scriptpubkeyman_tests.cpp index 90042f5252..d4997c418a 100644 --- a/src/wallet/test/scriptpubkeyman_tests.cpp +++ b/src/wallet/test/scriptpubkeyman_tests.cpp @@ -7,6 +7,7 @@ #include <test/util/setup_common.h> #include <wallet/scriptpubkeyman.h> #include <wallet/wallet.h> +#include <wallet/test/util.h> #include <boost/test/unit_test.hpp> @@ -18,7 +19,7 @@ BOOST_FIXTURE_TEST_SUITE(scriptpubkeyman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(CanProvide) { // Set up wallet and keyman variables. - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); LegacyScriptPubKeyMan& keyman = *wallet.GetOrCreateLegacyScriptPubKeyMan(); // Make a 1 of 2 multisig script diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index b7bf312edf..069ab25f26 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -7,7 +7,9 @@ #include <chain.h> #include <key.h> #include <key_io.h> +#include <streams.h> #include <test/util/setup_common.h> +#include <wallet/context.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> @@ -16,7 +18,7 @@ namespace wallet { std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key) { - auto wallet = std::make_unique<CWallet>(&chain, "", CreateMockWalletDatabase()); + auto wallet = std::make_unique<CWallet>(&chain, "", CreateMockableWalletDatabase()); { LOCK2(wallet->cs_wallet, ::cs_main); wallet->SetLastBlockProcessed(cchain.Height(), cchain.Tip()->GetBlockHash()); @@ -44,28 +46,39 @@ std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cc return wallet; } -std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database, DatabaseOptions& options) +std::shared_ptr<CWallet> TestLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, uint64_t create_flags) { - auto new_database = CreateMockWalletDatabase(options); - - // Get a cursor to the original database - auto batch = database.MakeBatch(); - std::unique_ptr<wallet::DatabaseCursor> cursor = batch->GetNewCursor(); + bilingual_str error; + std::vector<bilingual_str> warnings; + auto wallet = CWallet::Create(context, "", std::move(database), create_flags, error, warnings); + NotifyWalletLoaded(context, wallet); + if (context.chain) { + wallet->postInitProcess(); + } + return wallet; +} - // Get a batch for the new database - auto new_batch = new_database->MakeBatch(); +std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) +{ + DatabaseOptions options; + options.create_flags = WALLET_FLAG_DESCRIPTORS; + DatabaseStatus status; + bilingual_str error; + std::vector<bilingual_str> warnings; + auto database = MakeWalletDatabase("", options, status, error); + return TestLoadWallet(std::move(database), context, options.create_flags); +} - // Read all records from the original database and write them to the new one - while (true) { - DataStream key{}; - DataStream value{}; - DatabaseCursor::Status status = cursor->Next(key, value); - assert(status != DatabaseCursor::Status::FAIL); - if (status == DatabaseCursor::Status::DONE) break; - new_batch->Write(key, value); - } +void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet) +{ + SyncWithValidationInterfaceQueue(); + wallet->m_chain_notifications_handler.reset(); + UnloadWallet(std::move(wallet)); +} - return new_database; +std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database) +{ + return std::make_unique<MockableDatabase>(dynamic_cast<MockableDatabase&>(database).m_records); } std::string getnewaddress(CWallet& w) @@ -79,4 +92,107 @@ CTxDestination getNewDestination(CWallet& w, OutputType output_type) return *Assert(w.GetNewDestination(output_type, "")); } +// BytePrefix compares equality with other byte spans that begin with the same prefix. +struct BytePrefix { Span<const std::byte> prefix; }; +bool operator<(BytePrefix a, Span<const std::byte> b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); } +bool operator<(Span<const std::byte> a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; } + +MockableCursor::MockableCursor(const MockableData& records, bool pass, Span<const std::byte> prefix) +{ + m_pass = pass; + std::tie(m_cursor, m_cursor_end) = records.equal_range(BytePrefix{prefix}); +} + +DatabaseCursor::Status MockableCursor::Next(DataStream& key, DataStream& value) +{ + if (!m_pass) { + return Status::FAIL; + } + if (m_cursor == m_cursor_end) { + return Status::DONE; + } + key.clear(); + value.clear(); + const auto& [key_data, value_data] = *m_cursor; + key.write(key_data); + value.write(value_data); + m_cursor++; + return Status::MORE; +} + +bool MockableBatch::ReadKey(DataStream&& key, DataStream& value) +{ + if (!m_pass) { + return false; + } + SerializeData key_data{key.begin(), key.end()}; + const auto& it = m_records.find(key_data); + if (it == m_records.end()) { + return false; + } + value.clear(); + value.write(it->second); + return true; +} + +bool MockableBatch::WriteKey(DataStream&& key, DataStream&& value, bool overwrite) +{ + if (!m_pass) { + return false; + } + SerializeData key_data{key.begin(), key.end()}; + SerializeData value_data{value.begin(), value.end()}; + auto [it, inserted] = m_records.emplace(key_data, value_data); + if (!inserted && overwrite) { // Overwrite if requested + it->second = value_data; + inserted = true; + } + return inserted; +} + +bool MockableBatch::EraseKey(DataStream&& key) +{ + if (!m_pass) { + return false; + } + SerializeData key_data{key.begin(), key.end()}; + m_records.erase(key_data); + return true; +} + +bool MockableBatch::HasKey(DataStream&& key) +{ + if (!m_pass) { + return false; + } + SerializeData key_data{key.begin(), key.end()}; + return m_records.count(key_data) > 0; +} + +bool MockableBatch::ErasePrefix(Span<const std::byte> prefix) +{ + if (!m_pass) { + return false; + } + auto it = m_records.begin(); + while (it != m_records.end()) { + auto& key = it->first; + if (key.size() < prefix.size() || std::search(key.begin(), key.end(), prefix.begin(), prefix.end()) != key.begin()) { + it++; + continue; + } + it = m_records.erase(it); + } + return true; +} + +std::unique_ptr<WalletDatabase> CreateMockableWalletDatabase(MockableData records) +{ + return std::make_unique<MockableDatabase>(records); +} + +MockableDatabase& GetMockableDatabase(CWallet& wallet) +{ + return dynamic_cast<MockableDatabase&>(wallet.GetDatabase()); +} } // namespace wallet diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h index d726517e21..2a1fe639de 100644 --- a/src/wallet/test/util.h +++ b/src/wallet/test/util.h @@ -6,6 +6,8 @@ #define BITCOIN_WALLET_TEST_UTIL_H #include <script/standard.h> +#include <wallet/db.h> + #include <memory> class ArgsManager; @@ -18,19 +20,112 @@ class Chain; namespace wallet { class CWallet; -struct DatabaseOptions; class WalletDatabase; +struct WalletContext; + +static const DatabaseFormat DATABASE_FORMATS[] = { +#ifdef USE_SQLITE + DatabaseFormat::SQLITE, +#endif +#ifdef USE_BDB + DatabaseFormat::BERKELEY, +#endif +}; + +const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key); +std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context); +std::shared_ptr<CWallet> TestLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, uint64_t create_flags); +void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet); + // Creates a copy of the provided database -std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database, DatabaseOptions& options); +std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database); /** Returns a new encoded destination from the wallet (hardcoded to BECH32) */ std::string getnewaddress(CWallet& w); /** Returns a new destination, of an specific type, from the wallet */ CTxDestination getNewDestination(CWallet& w, OutputType output_type); +using MockableData = std::map<SerializeData, SerializeData, std::less<>>; + +class MockableCursor: public DatabaseCursor +{ +public: + MockableData::const_iterator m_cursor; + MockableData::const_iterator m_cursor_end; + bool m_pass; + + explicit MockableCursor(const MockableData& records, bool pass) : m_cursor(records.begin()), m_cursor_end(records.end()), m_pass(pass) {} + MockableCursor(const MockableData& records, bool pass, Span<const std::byte> prefix); + ~MockableCursor() {} + + Status Next(DataStream& key, DataStream& value) override; +}; + +class MockableBatch : public DatabaseBatch +{ +private: + MockableData& m_records; + bool m_pass; + + bool ReadKey(DataStream&& key, DataStream& value) override; + bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite=true) override; + bool EraseKey(DataStream&& key) override; + bool HasKey(DataStream&& key) override; + bool ErasePrefix(Span<const std::byte> prefix) override; + +public: + explicit MockableBatch(MockableData& records, bool pass) : m_records(records), m_pass(pass) {} + ~MockableBatch() {} + + void Flush() override {} + void Close() override {} + + std::unique_ptr<DatabaseCursor> GetNewCursor() override + { + return std::make_unique<MockableCursor>(m_records, m_pass); + } + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override { + return std::make_unique<MockableCursor>(m_records, m_pass, prefix); + } + bool TxnBegin() override { return m_pass; } + bool TxnCommit() override { return m_pass; } + bool TxnAbort() override { return m_pass; } +}; + +/** A WalletDatabase whose contents and return values can be modified as needed for testing + **/ +class MockableDatabase : public WalletDatabase +{ +public: + MockableData m_records; + bool m_pass{true}; + + MockableDatabase(MockableData records = {}) : WalletDatabase(), m_records(records) {} + ~MockableDatabase() {}; + + void Open() override {} + void AddRef() override {} + void RemoveRef() override {} + + bool Rewrite(const char* pszSkip=nullptr) override { return m_pass; } + bool Backup(const std::string& strDest) const override { return m_pass; } + void Flush() override {} + void Close() override {} + bool PeriodicFlush() override { return m_pass; } + void IncrementUpdateCounter() override {} + void ReloadDbEnv() override {} + + std::string Filename() override { return "mockable"; } + std::string Format() override { return "mock"; } + std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override { return std::make_unique<MockableBatch>(m_records, m_pass); } +}; + +std::unique_ptr<WalletDatabase> CreateMockableWalletDatabase(MockableData records = {}); + +MockableDatabase& GetMockableDatabase(CWallet& wallet); } // namespace wallet #endif // BITCOIN_WALLET_TEST_UTIL_H diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index 2dd8f9ad33..57c538e4ef 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -2,15 +2,17 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <wallet/test/util.h> #include <wallet/test/wallet_test_fixture.h> #include <scheduler.h> +#include <util/chaintype.h> namespace wallet { -WalletTestingSetup::WalletTestingSetup(const std::string& chainName) - : TestingSetup(chainName), +WalletTestingSetup::WalletTestingSetup(const ChainType chainType) + : TestingSetup(chainType), m_wallet_loader{interfaces::MakeWalletLoader(*m_node.chain, *Assert(m_node.args))}, - m_wallet(m_node.chain.get(), "", CreateMockWalletDatabase()) + m_wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()) { m_wallet.LoadWallet(); m_chain_notifications_handler = m_node.chain->handleNotifications({ &m_wallet, [](CWallet*) {} }); diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index f1ef15a282..4aae02e075 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -10,6 +10,7 @@ #include <interfaces/chain.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/chaintype.h> #include <util/check.h> #include <wallet/wallet.h> @@ -19,7 +20,7 @@ namespace wallet { /** Testing setup and teardown for wallet. */ struct WalletTestingSetup : public TestingSetup { - explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + explicit WalletTestingSetup(const ChainType chainType = ChainType::MAIN); ~WalletTestingSetup(); std::unique_ptr<interfaces::WalletLoader> m_wallet_loader; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index da4b553394..65b02c267d 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -29,7 +29,6 @@ #include <univalue.h> using node::MAX_BLOCKFILE_SIZE; -using node::UnlinkPrunedFiles; namespace wallet { RPCHelpMan importmulti(); @@ -43,26 +42,6 @@ static_assert(WALLET_INCREMENTAL_RELAY_FEE >= DEFAULT_INCREMENTAL_RELAY_FEE, "wa BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) -static std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) -{ - DatabaseOptions options; - options.create_flags = WALLET_FLAG_DESCRIPTORS; - DatabaseStatus status; - bilingual_str error; - std::vector<bilingual_str> warnings; - auto database = MakeWalletDatabase("", options, status, error); - auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); - NotifyWalletLoaded(context, wallet); - return wallet; -} - -static void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet) -{ - SyncWithValidationInterfaceQueue(); - wallet->m_chain_notifications_handler.reset(); - UnloadWallet(std::move(wallet)); -} - static CMutableTransaction TestSimpleSpend(const CTransaction& from, uint32_t index, const CKey& key, const CScript& pubkey) { CMutableTransaction mtx; @@ -98,7 +77,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions fails to read an unknown start block. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); { LOCK(wallet.cs_wallet); LOCK(Assert(m_node.chainman)->GetMutex()); @@ -119,7 +98,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(m_node.chain.get(), "", CreateMockWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); { LOCK(wallet.cs_wallet); LOCK(Assert(m_node.chainman)->GetMutex()); @@ -159,12 +138,12 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) file_number = oldTip->GetBlockPos().nFile; Assert(m_node.chainman)->m_blockman.PruneOneBlockFile(file_number); } - UnlinkPrunedFiles({file_number}); + m_node.chainman->m_blockman.UnlinkPrunedFiles({file_number}); // Verify ScanForWalletTransactions only picks transactions in the new block // file. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); { LOCK(wallet.cs_wallet); LOCK(Assert(m_node.chainman)->GetMutex()); @@ -188,11 +167,11 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) file_number = newTip->GetBlockPos().nFile; Assert(m_node.chainman)->m_blockman.PruneOneBlockFile(file_number); } - UnlinkPrunedFiles({file_number}); + m_node.chainman->m_blockman.UnlinkPrunedFiles({file_number}); // Verify ScanForWalletTransactions scans no blocks. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); { LOCK(wallet.cs_wallet); LOCK(Assert(m_node.chainman)->GetMutex()); @@ -226,13 +205,13 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) file_number = oldTip->GetBlockPos().nFile; Assert(m_node.chainman)->m_blockman.PruneOneBlockFile(file_number); } - UnlinkPrunedFiles({file_number}); + m_node.chainman->m_blockman.UnlinkPrunedFiles({file_number}); // Verify importmulti RPC returns failure for a key whose creation time is // before the missing block, and success for a key whose creation time is // after. { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); WalletContext context; @@ -298,7 +277,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) { WalletContext context; context.args = &m_args; - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); @@ -321,7 +300,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME // were scanned, and no prior blocks were scanned. { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); @@ -355,7 +334,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // debit functions. BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); LOCK(wallet.cs_wallet); LOCK(Assert(m_node.chainman)->GetMutex()); @@ -427,15 +406,6 @@ BOOST_AUTO_TEST_CASE(ComputeTimeSmart) BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 5, 50, 600), 300); } -static const DatabaseFormat DATABASE_FORMATS[] = { -#ifdef USE_SQLITE - DatabaseFormat::SQLITE, -#endif -#ifdef USE_BDB - DatabaseFormat::BERKELEY, -#endif -}; - void TestLoadWallet(const std::string& name, DatabaseFormat format, std::function<void(std::shared_ptr<CWallet>)> f) { node::NodeContext node; @@ -709,7 +679,7 @@ BOOST_FIXTURE_TEST_CASE(BasicOutputTypesTest, ListCoinsTest) BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); @@ -717,7 +687,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "")); } { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet->SetMinVersion(FEATURE_LATEST); @@ -855,10 +825,11 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // Reload wallet and make sure new transactions are detected despite events // being blocked + // Loading will also ask for current mempool transactions wallet = TestLoadWallet(context); BOOST_CHECK(rescan_completed); - // AddToWallet events for block_tx and mempool_tx - BOOST_CHECK_EQUAL(addtx_count, 2); + // AddToWallet events for block_tx and mempool_tx (x2) + BOOST_CHECK_EQUAL(addtx_count, 3); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); @@ -872,7 +843,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); // AddToWallet events for block_tx and mempool_tx events are counted a // second time as the notification queue is processed - BOOST_CHECK_EQUAL(addtx_count, 4); + BOOST_CHECK_EQUAL(addtx_count, 5); TestUnloadWallet(std::move(wallet)); @@ -895,7 +866,9 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); }); wallet = TestLoadWallet(context); - BOOST_CHECK_EQUAL(addtx_count, 2); + // Since mempool transactions are requested at the end of loading, there will + // be 2 additional AddToWallet calls, one from the previous test, and a duplicate for mempool_tx + BOOST_CHECK_EQUAL(addtx_count, 2 + 2); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); @@ -951,62 +924,13 @@ BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) TestUnloadWallet(std::move(wallet)); } -class FailCursor : public DatabaseCursor -{ -public: - Status Next(DataStream& key, DataStream& value) override { return Status::FAIL; } -}; - -/** RAII class that provides access to a FailDatabase. Which fails if needed. */ -class FailBatch : public DatabaseBatch -{ -private: - bool m_pass{true}; - bool ReadKey(DataStream&& key, DataStream& value) override { return m_pass; } - bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return m_pass; } - bool EraseKey(DataStream&& key) override { return m_pass; } - bool HasKey(DataStream&& key) override { return m_pass; } - bool ErasePrefix(Span<const std::byte> prefix) override { return m_pass; } - -public: - explicit FailBatch(bool pass) : m_pass(pass) {} - void Flush() override {} - void Close() override {} - - std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<FailCursor>(); } - bool TxnBegin() override { return false; } - bool TxnCommit() override { return false; } - bool TxnAbort() override { return false; } -}; - -/** A dummy WalletDatabase that does nothing, only fails if needed.**/ -class FailDatabase : public WalletDatabase -{ -public: - bool m_pass{true}; // false when this db should fail - - void Open() override {}; - void AddRef() override {} - void RemoveRef() override {} - bool Rewrite(const char* pszSkip=nullptr) override { return true; } - bool Backup(const std::string& strDest) const override { return true; } - void Close() override {} - void Flush() override {} - bool PeriodicFlush() override { return true; } - void IncrementUpdateCounter() override { ++nUpdateCounter; } - void ReloadDbEnv() override {} - std::string Filename() override { return "faildb"; } - std::string Format() override { return "faildb"; } - std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override { return std::make_unique<FailBatch>(m_pass); } -}; - /** * Checks a wallet invalid state where the inputs (prev-txs) of a new arriving transaction are not marked dirty, * while the transaction that spends them exist inside the in-memory wallet tx map (not stored on db due a db write failure). */ BOOST_FIXTURE_TEST_CASE(wallet_sync_tx_invalid_state_test, TestingSetup) { - CWallet wallet(m_node.chain.get(), "", std::make_unique<FailDatabase>()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -1014,11 +938,10 @@ BOOST_FIXTURE_TEST_CASE(wallet_sync_tx_invalid_state_test, TestingSetup) } // Add tx to wallet - const auto& op_dest = wallet.GetNewDestination(OutputType::BECH32M, ""); - BOOST_ASSERT(op_dest); + const auto op_dest{*Assert(wallet.GetNewDestination(OutputType::BECH32M, ""))}; CMutableTransaction mtx; - mtx.vout.push_back({COIN, GetScriptForDestination(*op_dest)}); + mtx.vout.push_back({COIN, GetScriptForDestination(op_dest)}); mtx.vin.push_back(CTxIn(g_insecure_rand_ctx.rand256(), 0)); const auto& tx_id_to_spend = wallet.AddToWallet(MakeTransactionRef(mtx), TxStateInMempool{})->GetHash(); @@ -1053,7 +976,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_sync_tx_invalid_state_test, TestingSetup) // 1) Make db always fail // 2) Try to add a transaction that spends the previously created transaction and // verify that we are not moving forward if the wallet cannot store it - static_cast<FailDatabase&>(wallet.GetDatabase()).m_pass = false; + GetMockableDatabase(wallet).m_pass = false; mtx.vin.clear(); mtx.vin.push_back(CTxIn(good_tx_id, 0)); BOOST_CHECK_EXCEPTION(wallet.transactionAddedToMempool(MakeTransactionRef(mtx)), diff --git a/src/wallet/test/walletdb_tests.cpp b/src/wallet/test/walletdb_tests.cpp index 21842fe780..17b6c4f7ed 100644 --- a/src/wallet/test/walletdb_tests.cpp +++ b/src/wallet/test/walletdb_tests.cpp @@ -6,6 +6,8 @@ #include <clientversion.h> #include <streams.h> #include <uint256.h> +#include <wallet/test/util.h> +#include <wallet/wallet.h> #include <boost/test/unit_test.hpp> @@ -27,5 +29,31 @@ BOOST_AUTO_TEST_CASE(walletdb_readkeyvalue) BOOST_CHECK_THROW(ssValue >> dummy, std::ios_base::failure); } +BOOST_AUTO_TEST_CASE(walletdb_read_write_deadlock) +{ + // Exercises a db read write operation that shouldn't deadlock. + for (const DatabaseFormat& db_format : DATABASE_FORMATS) { + // Context setup + DatabaseOptions options; + options.require_format = db_format; + DatabaseStatus status; + bilingual_str error_string; + std::unique_ptr<WalletDatabase> db = MakeDatabase(m_path_root / strprintf("wallet_%d_.dat", db_format).c_str(), options, status, error_string); + BOOST_CHECK_EQUAL(status, DatabaseStatus::SUCCESS); + + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", std::move(db))); + wallet->m_keypool_size = 4; + + // Create legacy spkm + LOCK(wallet->cs_wallet); + auto legacy_spkm = wallet->GetOrCreateLegacyScriptPubKeyMan(); + BOOST_CHECK(legacy_spkm->SetupGeneration(true)); + wallet->Flush(); + + // Now delete all records, which performs a read write operation. + BOOST_CHECK(wallet->GetLegacyScriptPubKeyMan()->DeleteRecords()); + } +} + BOOST_AUTO_TEST_SUITE_END() } // namespace wallet diff --git a/src/wallet/test/walletload_tests.cpp b/src/wallet/test/walletload_tests.cpp index 9f5a4b14d3..73a4b77188 100644 --- a/src/wallet/test/walletload_tests.cpp +++ b/src/wallet/test/walletload_tests.cpp @@ -34,7 +34,7 @@ public: BOOST_FIXTURE_TEST_CASE(wallet_load_unknown_descriptor, TestingSetup) { - std::unique_ptr<WalletDatabase> database = CreateMockWalletDatabase(); + std::unique_ptr<WalletDatabase> database = CreateMockableWalletDatabase(); { // Write unknown active descriptor WalletBatch batch(*database, false); @@ -70,38 +70,45 @@ bool HasAnyRecordOfType(WalletDatabase& db, const std::string& key) return false; } -BOOST_FIXTURE_TEST_CASE(wallet_load_verif_crypted_key_checksum, TestingSetup) +template<typename... Args> +SerializeData MakeSerializeData(const Args&... args) { - // The test duplicates the db so each case has its own db instance. - int NUMBER_OF_TESTS = 4; - std::vector<std::unique_ptr<WalletDatabase>> dbs; - CKey first_key; - auto get_db = [](std::vector<std::unique_ptr<WalletDatabase>>& dbs) { - std::unique_ptr<WalletDatabase> db = std::move(dbs.back()); - dbs.pop_back(); - return db; - }; - - { // Context setup. + CDataStream s(0, 0); + SerializeMany(s, args...); + return {s.begin(), s.end()}; +} + + +BOOST_FIXTURE_TEST_CASE(wallet_load_ckey, TestingSetup) +{ + SerializeData ckey_record_key; + SerializeData ckey_record_value; + MockableData records; + + { + // Context setup. // Create and encrypt legacy wallet - std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockWalletDatabase())); + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockableWalletDatabase())); LOCK(wallet->cs_wallet); auto legacy_spkm = wallet->GetOrCreateLegacyScriptPubKeyMan(); BOOST_CHECK(legacy_spkm->SetupGeneration(true)); - // Get the first key in the wallet + // Retrieve a key CTxDestination dest = *Assert(legacy_spkm->GetNewDestination(OutputType::LEGACY)); CKeyID key_id = GetKeyForDestination(*legacy_spkm, dest); + CKey first_key; BOOST_CHECK(legacy_spkm->GetKey(key_id, first_key)); - // Encrypt the wallet and duplicate database + // Encrypt the wallet BOOST_CHECK(wallet->EncryptWallet("encrypt")); wallet->Flush(); - DatabaseOptions options; - for (int i=0; i < NUMBER_OF_TESTS; i++) { - dbs.emplace_back(DuplicateMockDatabase(wallet->GetDatabase(), options)); - } + // Store a copy of all the records + records = GetMockableDatabase(*wallet).m_records; + + // Get the record for the retrieved key + ckey_record_key = MakeSerializeData(DBKeys::CRYPTED_KEY, first_key.GetPubKey()); + ckey_record_value = records.at(ckey_record_key); } { @@ -112,7 +119,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_load_verif_crypted_key_checksum, TestingSetup) // the records every time that 'CWallet::Unlock' gets called, which is not good. // Load the wallet and check that is encrypted - std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", get_db(dbs))); + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockableWalletDatabase(records))); BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::LOAD_OK); BOOST_CHECK(wallet->IsCrypted()); BOOST_CHECK(HasAnyRecordOfType(wallet->GetDatabase(), DBKeys::CRYPTED_KEY)); @@ -127,18 +134,12 @@ BOOST_FIXTURE_TEST_CASE(wallet_load_verif_crypted_key_checksum, TestingSetup) { // Second test case: // Verify that loading up a 'ckey' with no checksum triggers a complete re-write of the crypted keys. - std::unique_ptr<WalletDatabase> db = get_db(dbs); - { - std::unique_ptr<DatabaseBatch> batch = db->MakeBatch(false); - std::pair<std::vector<unsigned char>, uint256> value; - BOOST_CHECK(batch->Read(std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()), value)); - const auto key = std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()); - BOOST_CHECK(batch->Write(key, value.first, /*fOverwrite=*/true)); - } + // Cut off the 32 byte checksum from a ckey record + records[ckey_record_key].resize(ckey_record_value.size() - 32); // Load the wallet and check that is encrypted - std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", std::move(db))); + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockableWalletDatabase(records))); BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::LOAD_OK); BOOST_CHECK(wallet->IsCrypted()); BOOST_CHECK(HasAnyRecordOfType(wallet->GetDatabase(), DBKeys::CRYPTED_KEY)); @@ -154,35 +155,25 @@ BOOST_FIXTURE_TEST_CASE(wallet_load_verif_crypted_key_checksum, TestingSetup) { // Third test case: // Verify that loading up a 'ckey' with an invalid checksum throws an error. - std::unique_ptr<WalletDatabase> db = get_db(dbs); - { - std::unique_ptr<DatabaseBatch> batch = db->MakeBatch(false); - std::vector<unsigned char> crypted_data; - BOOST_CHECK(batch->Read(std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()), crypted_data)); - - // Write an invalid checksum - std::pair<std::vector<unsigned char>, uint256> value = std::make_pair(crypted_data, uint256::ONE); - const auto key = std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()); - BOOST_CHECK(batch->Write(key, value, /*fOverwrite=*/true)); - } - - std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", std::move(db))); + + // Cut off the 32 byte checksum from a ckey record + records[ckey_record_key].resize(ckey_record_value.size() - 32); + // Fill in the checksum space with 0s + records[ckey_record_key].resize(ckey_record_value.size()); + + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockableWalletDatabase(records))); BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::CORRUPT); } { // Fourth test case: // Verify that loading up a 'ckey' with an invalid pubkey throws an error - std::unique_ptr<WalletDatabase> db = get_db(dbs); - { - CPubKey invalid_key; - BOOST_ASSERT(!invalid_key.IsValid()); - const auto key = std::make_pair(DBKeys::CRYPTED_KEY, invalid_key); - std::pair<std::vector<unsigned char>, uint256> value; - BOOST_CHECK(db->MakeBatch(false)->Write(key, value, /*fOverwrite=*/true)); - } - - std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", std::move(db))); + CPubKey invalid_key; + BOOST_CHECK(!invalid_key.IsValid()); + SerializeData key = MakeSerializeData(DBKeys::CRYPTED_KEY, invalid_key); + records[key] = ckey_record_value; + + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockableWalletDatabase(records))); BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::CORRUPT); } } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index caf95a3f03..62f0f53b01 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -42,6 +42,7 @@ #include <wallet/context.h> #include <wallet/external_signer_scriptpubkeyman.h> #include <wallet/fees.h> +#include <wallet/scriptpubkeyman.h> #include <univalue.h> @@ -55,9 +56,9 @@ namespace wallet { bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) { - util::SettingsValue setting_value = chain.getRwSetting("wallet"); + common::SettingsValue setting_value = chain.getRwSetting("wallet"); if (!setting_value.isArray()) setting_value.setArray(); - for (const util::SettingsValue& value : setting_value.getValues()) { + for (const common::SettingsValue& value : setting_value.getValues()) { if (value.isStr() && value.get_str() == wallet_name) return true; } setting_value.push_back(wallet_name); @@ -66,10 +67,10 @@ bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) { - util::SettingsValue setting_value = chain.getRwSetting("wallet"); + common::SettingsValue setting_value = chain.getRwSetting("wallet"); if (!setting_value.isArray()) return true; - util::SettingsValue new_value(util::SettingsValue::VARR); - for (const util::SettingsValue& value : setting_value.getValues()) { + common::SettingsValue new_value(common::SettingsValue::VARR); + for (const common::SettingsValue& value : setting_value.getValues()) { if (!value.isStr() || value.get_str() != wallet_name) new_value.push_back(value); } if (new_value.size() == setting_value.size()) return true; @@ -559,13 +560,14 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, return false; if (Unlock(_vMasterKey)) { - int64_t nStartTime = GetTimeMillis(); + constexpr MillisecondsDouble target{100}; + auto start{SteadyClock::now()}; crypter.SetKeyFromPassphrase(strNewWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod); - pMasterKey.second.nDeriveIterations = static_cast<unsigned int>(pMasterKey.second.nDeriveIterations * (100 / ((double)(GetTimeMillis() - nStartTime)))); + pMasterKey.second.nDeriveIterations = static_cast<unsigned int>(pMasterKey.second.nDeriveIterations * target / (SteadyClock::now() - start)); - nStartTime = GetTimeMillis(); + start = SteadyClock::now(); crypter.SetKeyFromPassphrase(strNewWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod); - pMasterKey.second.nDeriveIterations = (pMasterKey.second.nDeriveIterations + static_cast<unsigned int>(pMasterKey.second.nDeriveIterations * 100 / ((double)(GetTimeMillis() - nStartTime)))) / 2; + pMasterKey.second.nDeriveIterations = (pMasterKey.second.nDeriveIterations + static_cast<unsigned int>(pMasterKey.second.nDeriveIterations * target / (SteadyClock::now() - start))) / 2; if (pMasterKey.second.nDeriveIterations < 25000) pMasterKey.second.nDeriveIterations = 25000; @@ -762,13 +764,14 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) GetStrongRandBytes(kMasterKey.vchSalt); CCrypter crypter; - int64_t nStartTime = GetTimeMillis(); + constexpr MillisecondsDouble target{100}; + auto start{SteadyClock::now()}; crypter.SetKeyFromPassphrase(strWalletPassphrase, kMasterKey.vchSalt, 25000, kMasterKey.nDerivationMethod); - kMasterKey.nDeriveIterations = static_cast<unsigned int>(2500000 / ((double)(GetTimeMillis() - nStartTime))); + kMasterKey.nDeriveIterations = static_cast<unsigned int>(25000 * target / (SteadyClock::now() - start)); - nStartTime = GetTimeMillis(); + start = SteadyClock::now(); crypter.SetKeyFromPassphrase(strWalletPassphrase, kMasterKey.vchSalt, kMasterKey.nDeriveIterations, kMasterKey.nDerivationMethod); - kMasterKey.nDeriveIterations = (kMasterKey.nDeriveIterations + static_cast<unsigned int>(kMasterKey.nDeriveIterations * 100 / ((double)(GetTimeMillis() - nStartTime)))) / 2; + kMasterKey.nDeriveIterations = (kMasterKey.nDeriveIterations + static_cast<unsigned int>(kMasterKey.nDeriveIterations * target / (SteadyClock::now() - start))) / 2; if (kMasterKey.nDeriveIterations < 25000) kMasterKey.nDeriveIterations = 25000; @@ -1264,11 +1267,6 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK(cs_wallet); - WalletBatch batch(GetDatabase()); - - std::set<uint256> todo; - std::set<uint256> done; - // Can't mark abandoned if confirmed or in mempool auto it = mapWallet.find(hashTx); assert(it != mapWallet.end()); @@ -1277,44 +1275,25 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) return false; } - todo.insert(hashTx); - - while (!todo.empty()) { - uint256 now = *todo.begin(); - todo.erase(now); - done.insert(now); - auto it = mapWallet.find(now); - assert(it != mapWallet.end()); - CWalletTx& wtx = it->second; - int currentconfirm = GetTxDepthInMainChain(wtx); - // If the orig tx was not in block, none of its spends can be - assert(currentconfirm <= 0); - // if (currentconfirm < 0) {Tx and spends are already conflicted, no need to abandon} - if (currentconfirm == 0 && !wtx.isAbandoned()) { - // If the orig tx was not in block/mempool, none of its spends can be in mempool - assert(!wtx.InMempool()); + auto try_updating_state = [](CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + // If the orig tx was not in block/mempool, none of its spends can be. + assert(!wtx.isConfirmed()); + assert(!wtx.InMempool()); + // If already conflicted or abandoned, no need to set abandoned + if (!wtx.isConflicted() && !wtx.isAbandoned()) { wtx.m_state = TxStateInactive{/*abandoned=*/true}; - wtx.MarkDirty(); - batch.WriteTx(wtx); - NotifyTransactionChanged(wtx.GetHash(), CT_UPDATED); - // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too. - // States are not permanent, so these transactions can become unabandoned if they are re-added to the - // mempool, or confirmed in a block, or conflicted. - // Note: If the reorged coinbase is re-added to the main chain, the descendants that have not had their - // states change will remain abandoned and will require manual broadcast if the user wants them. - for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { - std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(now, i)); - for (TxSpends::const_iterator iter = range.first; iter != range.second; ++iter) { - if (!done.count(iter->second)) { - todo.insert(iter->second); - } - } - } - // If a transaction changes 'conflicted' state, that changes the balance - // available of the outputs it spends. So force those to be recomputed - MarkInputsDirty(wtx.tx); + return TxUpdate::NOTIFY_CHANGED; } - } + return TxUpdate::UNCHANGED; + }; + + // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too. + // States are not permanent, so these transactions can become unabandoned if they are re-added to the + // mempool, or confirmed in a block, or conflicted. + // Note: If the reorged coinbase is re-added to the main chain, the descendants that have not had their + // states change will remain abandoned and will require manual broadcast if the user wants them. + + RecursiveUpdateTxState(hashTx, try_updating_state); return true; } @@ -1331,13 +1310,29 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c if (conflictconfirms >= 0) return; + auto try_updating_state = [&](CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + if (conflictconfirms < GetTxDepthInMainChain(wtx)) { + // Block is 'more conflicted' than current confirm; update. + // Mark transaction as conflicted with this block. + wtx.m_state = TxStateConflicted{hashBlock, conflicting_height}; + return TxUpdate::CHANGED; + } + return TxUpdate::UNCHANGED; + }; + + // Iterate over all its outputs, and mark transactions in the wallet that spend them conflicted too. + RecursiveUpdateTxState(hashTx, try_updating_state); + +} + +void CWallet::RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { // Do not flush the wallet here for performance reasons WalletBatch batch(GetDatabase(), false); std::set<uint256> todo; std::set<uint256> done; - todo.insert(hashTx); + todo.insert(tx_hash); while (!todo.empty()) { uint256 now = *todo.begin(); @@ -1346,14 +1341,12 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = GetTxDepthInMainChain(wtx); - if (conflictconfirms < currentconfirm) { - // Block is 'more conflicted' than current confirm; update. - // Mark transaction as conflicted with this block. - wtx.m_state = TxStateConflicted{hashBlock, conflicting_height}; + + TxUpdate update_state = try_updating_state(wtx); + if (update_state != TxUpdate::UNCHANGED) { wtx.MarkDirty(); batch.WriteTx(wtx); - // Iterate over all its outputs, and mark transactions in the wallet that spend them conflicted too + // Iterate over all its outputs, and update those tx states as well (if applicable) for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(now, i)); for (TxSpends::const_iterator iter = range.first; iter != range.second; ++iter) { @@ -1362,7 +1355,12 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c } } } - // If a transaction changes 'conflicted' state, that changes the balance + + if (update_state == TxUpdate::NOTIFY_CHANGED) { + NotifyTransactionChanged(wtx.GetHash(), CT_UPDATED); + } + + // If a transaction changes its tx state, that usually changes the balance // available of the outputs it spends. So force those to be recomputed MarkInputsDirty(wtx.tx); } @@ -1434,6 +1432,12 @@ void CWallet::blockConnected(const interfaces::BlockInfo& block) m_last_block_processed_height = block.height; m_last_block_processed = block.hash; + + // No need to scan block if it was created before the wallet birthday. + // Uses chain max time and twice the grace period to adjust time for block time variability. + if (block.chain_time_max < m_birth_time.load() - (TIMESTAMP_WINDOW * 2)) return; + + // Scan block for (size_t index = 0; index < block.data->vtx.size(); index++) { SyncTransaction(block.data->vtx[index], TxStateConfirmed{block.hash, block.height, static_cast<int>(index)}); transactionRemovedFromMempool(block.data->vtx[index], MemPoolRemovalReason::BLOCK); @@ -1451,8 +1455,36 @@ void CWallet::blockDisconnected(const interfaces::BlockInfo& block) // future with a stickier abandoned state or even removing abandontransaction call. m_last_block_processed_height = block.height - 1; m_last_block_processed = *Assert(block.prev_hash); + + int disconnect_height = block.height; + for (const CTransactionRef& ptx : Assert(block.data)->vtx) { SyncTransaction(ptx, TxStateInactive{}); + + for (const CTxIn& tx_in : ptx->vin) { + // No other wallet transactions conflicted with this transaction + if (mapTxSpends.count(tx_in.prevout) < 1) continue; + + std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(tx_in.prevout); + + // For all of the spends that conflict with this transaction + for (TxSpends::const_iterator _it = range.first; _it != range.second; ++_it) { + CWalletTx& wtx = mapWallet.find(_it->second)->second; + + if (!wtx.isConflicted()) continue; + + auto try_updating_state = [&](CWalletTx& tx) { + if (!tx.isConflicted()) return TxUpdate::UNCHANGED; + if (tx.state<TxStateConflicted>()->conflicting_block_height >= disconnect_height) { + tx.m_state = TxStateInactive{}; + return TxUpdate::CHANGED; + } + return TxUpdate::UNCHANGED; + }; + + RecursiveUpdateTxState(wtx.tx->GetHash(), try_updating_state); + } + } } } @@ -1777,6 +1809,14 @@ bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScri return true; } +void CWallet::FirstKeyTimeChanged(const ScriptPubKeyMan* spkm, int64_t new_birth_time) +{ + int64_t birthtime = m_birth_time.load(); + if (new_birth_time < birthtime) { + m_birth_time = new_birth_time; + } +} + /** * Scan active chain for relevant transactions after importing keys. This should * be called whenever new keys are added to the wallet, with the oldest key @@ -2792,7 +2832,7 @@ bool CWallet::SetAddressPreviouslySpent(WalletBatch& batch, const CTxDestination return false; if (!used) { - if (auto* data{util::FindKey(m_address_book, dest)}) data->previously_spent = false; + if (auto* data{common::FindKey(m_address_book, dest)}) data->previously_spent = false; return batch.WriteAddressPreviouslySpent(dest, false); } @@ -2812,7 +2852,7 @@ void CWallet::LoadAddressReceiveRequest(const CTxDestination& dest, const std::s bool CWallet::IsAddressPreviouslySpent(const CTxDestination& dest) const { - if (auto* data{util::FindKey(m_address_book, dest)}) return data->previously_spent; + if (auto* data{common::FindKey(m_address_book, dest)}) return data->previously_spent; return false; } @@ -3105,6 +3145,14 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri // Try to top up keypool. No-op if the wallet is locked. walletInstance->TopUpKeyPool(); + // Cache the first key time + std::optional<int64_t> time_first_key; + for (auto spk_man : walletInstance->GetAllScriptPubKeyMans()) { + int64_t time = spk_man->GetTimeFirstKey(); + if (!time_first_key || time < *time_first_key) time_first_key = time; + } + if (time_first_key) walletInstance->m_birth_time = *time_first_key; + if (chain && !AttachChain(walletInstance, *chain, rescan_required, error, warnings)) { return nullptr; } @@ -3180,11 +3228,7 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf { // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) - std::optional<int64_t> time_first_key; - for (auto spk_man : walletInstance->GetAllScriptPubKeyMans()) { - int64_t time = spk_man->GetTimeFirstKey(); - if (!time_first_key || time < *time_first_key) time_first_key = time; - } + std::optional<int64_t> time_first_key = walletInstance->m_birth_time.load(); if (time_first_key) { FoundBlock found = FoundBlock().height(rescan_height); chain.findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, found); @@ -3496,6 +3540,14 @@ LegacyScriptPubKeyMan* CWallet::GetOrCreateLegacyScriptPubKeyMan() return GetLegacyScriptPubKeyMan(); } +void CWallet::AddScriptPubKeyMan(const uint256& id, std::unique_ptr<ScriptPubKeyMan> spkm_man) +{ + const auto& spkm = m_spk_managers[id] = std::move(spkm_man); + + // Update birth time if needed + FirstKeyTimeChanged(spkm.get(), spkm->GetTimeFirstKey()); +} + void CWallet::SetupLegacyScriptPubKeyMan() { if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty() || IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { @@ -3507,7 +3559,8 @@ void CWallet::SetupLegacyScriptPubKeyMan() m_internal_spk_managers[type] = spk_manager.get(); m_external_spk_managers[type] = spk_manager.get(); } - m_spk_managers[spk_manager->GetID()] = std::move(spk_manager); + uint256 id = spk_manager->GetID(); + AddScriptPubKeyMan(id, std::move(spk_manager)); } const CKeyingMaterial& CWallet::GetEncryptionKey() const @@ -3525,6 +3578,7 @@ void CWallet::ConnectScriptPubKeyManNotifiers() for (const auto& spk_man : GetActiveScriptPubKeyMans()) { spk_man->NotifyWatchonlyChanged.connect(NotifyWatchonlyChanged); spk_man->NotifyCanGetAddressesChanged.connect(NotifyCanGetAddressesChanged); + spk_man->NotifyFirstKeyTimeChanged.connect(std::bind(&CWallet::FirstKeyTimeChanged, this, std::placeholders::_1, std::placeholders::_2)); } } @@ -3532,10 +3586,10 @@ void CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc) { if (IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this, desc, m_keypool_size)); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); } else { auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc, m_keypool_size)); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); } } @@ -3556,7 +3610,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) } spk_manager->SetupDescriptorGeneration(master_key, t, internal); uint256 id = spk_manager->GetID(); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); AddActiveScriptPubKeyMan(id, t, internal); } } @@ -3587,7 +3641,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() if (!signer_res.isObject()) throw std::runtime_error(std::string(__func__) + ": Unexpected result"); for (bool internal : {false, true}) { - const UniValue& descriptor_vals = find_value(signer_res, internal ? "internal" : "receive"); + const UniValue& descriptor_vals = signer_res.find_value(internal ? "internal" : "receive"); if (!descriptor_vals.isArray()) throw std::runtime_error(std::string(__func__) + ": Unexpected result"); for (const UniValue& desc_val : descriptor_vals.get_array().getValues()) { const std::string& desc_str = desc_val.getValStr(); @@ -3604,7 +3658,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() auto spk_manager = std::unique_ptr<ExternalSignerScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this, m_keypool_size)); spk_manager->SetupDescriptor(std::move(desc)); uint256 id = spk_manager->GetID(); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); AddActiveScriptPubKeyMan(id, t, internal); } } @@ -3721,7 +3775,8 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat spk_man = new_spk_man.get(); // Save the descriptor to memory - m_spk_managers[new_spk_man->GetID()] = std::move(new_spk_man); + uint256 id = new_spk_man->GetID(); + AddScriptPubKeyMan(id, std::move(new_spk_man)); } // Add the private keys to the descriptor @@ -3820,9 +3875,7 @@ bool CWallet::MigrateToSQLite(bilingual_str& error) bool began = batch->TxnBegin(); assert(began); // This is a critical error, the new db could not be written to. The original db exists as a backup, but we should not continue execution. for (const auto& [key, value] : records) { - DataStream ss_key{key}; - DataStream ss_value{value}; - if (!batch->Write(ss_key, ss_value)) { + if (!batch->Write(MakeUCharSpan(key), MakeUCharSpan(value))) { batch->TxnAbort(); m_database->Close(); fs::remove(m_database->Filename()); @@ -3864,7 +3917,8 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) error = _("Error: Duplicate descriptors created during migration. Your wallet may be corrupted."); return false; } - m_spk_managers[desc_spkm->GetID()] = std::move(desc_spkm); + uint256 id = desc_spkm->GetID(); + AddScriptPubKeyMan(id, std::move(desc_spkm)); } // Remove the LegacyScriptPubKeyMan from disk diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 5252b46cdc..cbd5008366 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -284,7 +284,7 @@ private: std::atomic<bool> fScanningWallet{false}; // controlled by WalletRescanReserver std::atomic<bool> m_attaching_chain{false}; std::atomic<bool> m_scanning_with_passphrase{false}; - std::atomic<int64_t> m_scanning_start{0}; + std::atomic<SteadyClock::time_point> m_scanning_start{SteadyClock::time_point{}}; std::atomic<double> m_scanning_progress{0}; friend class WalletRescanReserver; @@ -299,6 +299,10 @@ private: // Local time that the tip block was received. Used to schedule wallet rebroadcasts. std::atomic<int64_t> m_best_block_time {0}; + // First created key time. Used to skip blocks prior to this time. + // 'std::numeric_limits<int64_t>::max()' if wallet is blank. + std::atomic<int64_t> m_birth_time{std::numeric_limits<int64_t>::max()}; + /** * Used to keep track of spent outpoints, and * detect and report conflicts (double-spends or @@ -330,6 +334,13 @@ private: /** Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); + enum class TxUpdate { UNCHANGED, CHANGED, NOTIFY_CHANGED }; + + using TryUpdatingStateFn = std::function<TxUpdate(CWalletTx& wtx)>; + + /** Mark a transaction (and its in-wallet descendants) as a particular tx state. */ + void RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -380,6 +391,10 @@ private: // ScriptPubKeyMan::GetID. In many cases it will be the hash of an internal structure std::map<uint256, std::unique_ptr<ScriptPubKeyMan>> m_spk_managers; + // Appends spk managers into the main 'm_spk_managers'. + // Must be the only method adding data to it. + void AddScriptPubKeyMan(const uint256& id, std::unique_ptr<ScriptPubKeyMan> spkm_man); + /** * Catch wallet up to current chain, scanning new blocks, updating the best * block locator and m_last_block_processed, and registering for @@ -505,7 +520,7 @@ public: bool IsAbortingRescan() const { return fAbortRescan; } bool IsScanning() const { return fScanningWallet; } bool IsScanningWithPassphrase() const { return m_scanning_with_passphrase; } - int64_t ScanningDuration() const { return fScanningWallet ? GetTimeMillis() - m_scanning_start : 0; } + SteadyClock::duration ScanningDuration() const { return fScanningWallet ? SteadyClock::now() - m_scanning_start.load() : SteadyClock::duration{}; } double ScanningProgress() const { return fScanningWallet ? (double) m_scanning_progress : 0; } //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo @@ -641,6 +656,9 @@ public: bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool apply_label, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** Updates wallet birth time if 'new_birth_time' is below it */ + void FirstKeyTimeChanged(const ScriptPubKeyMan* spkm, int64_t new_birth_time); + CFeeRate m_pay_tx_fee{DEFAULT_PAY_TX_FEE}; unsigned int m_confirm_target{DEFAULT_TX_CONFIRM_TARGET}; /** Allow Coin Selection to pick unconfirmed UTXOs that were sent from our own wallet if it @@ -1014,7 +1032,7 @@ public: return false; } m_wallet.m_scanning_with_passphrase.exchange(with_passphrase); - m_wallet.m_scanning_start = GetTimeMillis(); + m_wallet.m_scanning_start = SteadyClock::now(); m_wallet.m_scanning_progress = 0; m_could_reserve = true; return true; @@ -1056,7 +1074,7 @@ struct MigrationResult { }; //! Do all steps to migrate a legacy wallet to a descriptor wallet -util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& wallet_name, const SecureString& passphrase, WalletContext& context); +[[nodiscard]] util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& wallet_name, const SecureString& passphrase, WalletContext& context); } // namespace wallet #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 005592d720..34fe8ab17f 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -5,13 +5,13 @@ #include <wallet/walletdb.h> +#include <common/system.h> #include <key_io.h> #include <protocol.h> #include <serialize.h> #include <sync.h> #include <util/bip32.h> #include <util/fs.h> -#include <util/system.h> #include <util/time.h> #include <util/translation.h> #ifdef USE_BDB @@ -1136,6 +1136,9 @@ bool WalletBatch::WriteWalletFlags(const uint64_t flags) bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) { + // Begin db txn + if (!m_batch->TxnBegin()) return false; + // Get cursor std::unique_ptr<DatabaseCursor> cursor = m_batch->GetNewCursor(); if (!cursor) @@ -1144,8 +1147,7 @@ bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) } // Iterate the DB and look for any records that have the type prefixes - while (true) - { + while (true) { // Read next record DataStream key{}; DataStream value{}; @@ -1153,6 +1155,8 @@ bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) if (status == DatabaseCursor::Status::DONE) { break; } else if (status == DatabaseCursor::Status::FAIL) { + cursor.reset(nullptr); + m_batch->TxnAbort(); // abort db txn return false; } @@ -1163,10 +1167,16 @@ bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) key >> type; if (types.count(type) > 0) { - m_batch->Erase(key_data); + if (!m_batch->Erase(key_data)) { + cursor.reset(nullptr); + m_batch->TxnAbort(); + return false; // erase failed + } } } - return true; + // Finish db txn + cursor.reset(nullptr); + return m_batch->TxnCommit(); } bool WalletBatch::TxnBegin() @@ -1262,44 +1272,4 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } - -/** Return object for accessing dummy database with no read/write capabilities. */ -std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase() -{ - return std::make_unique<DummyDatabase>(); -} - -/** Return object for accessing temporary in-memory database. */ -std::unique_ptr<WalletDatabase> CreateMockWalletDatabase(DatabaseOptions& options) -{ - - std::optional<DatabaseFormat> format; - if (options.require_format) format = options.require_format; - if (!format) { -#ifdef USE_BDB - format = DatabaseFormat::BERKELEY; -#endif -#ifdef USE_SQLITE - format = DatabaseFormat::SQLITE; -#endif - } - - if (format == DatabaseFormat::SQLITE) { -#ifdef USE_SQLITE - return std::make_unique<SQLiteDatabase>(":memory:", "", options, true); -#endif - assert(false); - } - -#ifdef USE_BDB - return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), "", options); -#endif - assert(false); -} - -std::unique_ptr<WalletDatabase> CreateMockWalletDatabase() -{ - DatabaseOptions options; - return CreateMockWalletDatabase(options); -} } // namespace wallet diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 72086e950a..f84a89b23f 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -305,13 +305,6 @@ using KeyFilterFn = std::function<bool(const std::string&)>; //! Unserialize a given Key-Value pair and load it into the wallet bool ReadKeyValue(CWallet* pwallet, DataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn = nullptr); - -/** Return object for accessing dummy database with no read/write capabilities. */ -std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase(); - -/** Return object for accessing temporary in-memory database. */ -std::unique_ptr<WalletDatabase> CreateMockWalletDatabase(DatabaseOptions& options); -std::unique_ptr<WalletDatabase> CreateMockWalletDatabase(); } // namespace wallet #endif // BITCOIN_WALLET_WALLETDB_H diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 37847b64ef..c5975144c1 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -112,20 +112,6 @@ public: WalletDescriptor() {} WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) {} }; - -class CWallet; -class DescriptorScriptPubKeyMan; - -/** struct containing information needed for migrating legacy wallets to descriptor wallets */ -struct MigrationData -{ - CExtKey master_key; - std::vector<std::pair<std::string, int64_t>> watch_descs; - std::vector<std::pair<std::string, int64_t>> solvable_descs; - std::vector<std::unique_ptr<DescriptorScriptPubKeyMan>> desc_spkms; - std::shared_ptr<CWallet> watchonly_wallet{nullptr}; - std::shared_ptr<CWallet> solvable_wallet{nullptr}; -}; } // namespace wallet #endif // BITCOIN_WALLET_WALLETUTIL_H diff --git a/src/warnings.cpp b/src/warnings.cpp index d0de706953..cb73c7aea2 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -5,9 +5,9 @@ #include <warnings.h> +#include <common/system.h> #include <sync.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <vector> diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h index cf0ee48f47..17fa7bbaa9 100644 --- a/src/zmq/zmqabstractnotifier.h +++ b/src/zmq/zmqabstractnotifier.h @@ -6,6 +6,7 @@ #define BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H #include <cstdint> +#include <functional> #include <memory> #include <string> @@ -13,7 +14,7 @@ class CBlockIndex; class CTransaction; class CZMQAbstractNotifier; -using CZMQNotifierFactory = std::unique_ptr<CZMQAbstractNotifier> (*)(); +using CZMQNotifierFactory = std::function<std::unique_ptr<CZMQAbstractNotifier>()>; class CZMQAbstractNotifier { diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 9920d80a69..6755368249 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -39,12 +39,14 @@ std::list<const CZMQAbstractNotifier*> CZMQNotificationInterface::GetActiveNotif return result; } -CZMQNotificationInterface* CZMQNotificationInterface::Create() +std::unique_ptr<CZMQNotificationInterface> CZMQNotificationInterface::Create(std::function<bool(CBlock&, const CBlockIndex&)> get_block_by_index) { std::map<std::string, CZMQNotifierFactory> factories; factories["pubhashblock"] = CZMQAbstractNotifier::Create<CZMQPublishHashBlockNotifier>; factories["pubhashtx"] = CZMQAbstractNotifier::Create<CZMQPublishHashTransactionNotifier>; - factories["pubrawblock"] = CZMQAbstractNotifier::Create<CZMQPublishRawBlockNotifier>; + factories["pubrawblock"] = [&get_block_by_index]() -> std::unique_ptr<CZMQAbstractNotifier> { + return std::make_unique<CZMQPublishRawBlockNotifier>(get_block_by_index); + }; factories["pubrawtx"] = CZMQAbstractNotifier::Create<CZMQPublishRawTransactionNotifier>; factories["pubsequence"] = CZMQAbstractNotifier::Create<CZMQPublishSequenceNotifier>; @@ -68,7 +70,7 @@ CZMQNotificationInterface* CZMQNotificationInterface::Create() notificationInterface->notifiers = std::move(notifiers); if (notificationInterface->Initialize()) { - return notificationInterface.release(); + return notificationInterface; } } @@ -198,4 +200,4 @@ void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CB }); } -CZMQNotificationInterface* g_zmq_notification_interface = nullptr; +std::unique_ptr<CZMQNotificationInterface> g_zmq_notification_interface; diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index a43f9bfef3..ce67633b30 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -9,6 +9,7 @@ #include <validationinterface.h> #include <cstdint> +#include <functional> #include <list> #include <memory> @@ -23,7 +24,7 @@ public: std::list<const CZMQAbstractNotifier*> GetActiveNotifiers() const; - static CZMQNotificationInterface* Create(); + static std::unique_ptr<CZMQNotificationInterface> Create(std::function<bool(CBlock&, const CBlockIndex&)> get_block_by_index); protected: bool Initialize(); @@ -43,6 +44,6 @@ private: std::list<std::unique_ptr<CZMQAbstractNotifier>> notifiers; }; -extern CZMQNotificationInterface* g_zmq_notification_interface; +extern std::unique_ptr<CZMQNotificationInterface> g_zmq_notification_interface; #endif // BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 55f3d4e934..1241431523 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -39,8 +39,6 @@ namespace Consensus { struct Params; } -using node::ReadBlockFromDisk; - static std::multimap<std::string, CZMQAbstractPublishNotifier*> mapPublishNotifiers; static const char *MSG_HASHBLOCK = "hashblock"; @@ -99,9 +97,8 @@ static bool IsZMQAddressIPV6(const std::string &zmq_address) const size_t colon_index = zmq_address.rfind(':'); if (tcp_index == 0 && colon_index != std::string::npos) { const std::string ip = zmq_address.substr(tcp_prefix.length(), colon_index - tcp_prefix.length()); - CNetAddr addr; - LookupHost(ip, addr, false); - if (addr.IsIPv6()) return true; + const std::optional<CNetAddr> addr{LookupHost(ip, false)}; + if (addr.has_value() && addr.value().IsIPv6()) return true; } return false; } @@ -247,10 +244,9 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { LogPrint(BCLog::ZMQ, "Publish rawblock %s to %s\n", pindex->GetBlockHash().GetHex(), this->address); - const Consensus::Params& consensusParams = Params().GetConsensus(); CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); CBlock block; - if (!ReadBlockFromDisk(block, pindex, consensusParams)) { + if (!m_get_block_by_index(block, *pindex)) { zmqError("Can't read block from disk"); return false; } diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h index 18336a5eb0..a5cd433761 100644 --- a/src/zmq/zmqpublishnotifier.h +++ b/src/zmq/zmqpublishnotifier.h @@ -9,7 +9,9 @@ #include <cstddef> #include <cstdint> +#include <functional> +class CBlock; class CBlockIndex; class CTransaction; @@ -46,7 +48,12 @@ public: class CZMQPublishRawBlockNotifier : public CZMQAbstractPublishNotifier { +private: + const std::function<bool(CBlock&, const CBlockIndex&)> m_get_block_by_index; + public: + CZMQPublishRawBlockNotifier(std::function<bool(CBlock&, const CBlockIndex&)> get_block_by_index) + : m_get_block_by_index{std::move(get_block_by_index)} {} bool NotifyBlock(const CBlockIndex *pindex) override; }; diff --git a/test/README.md b/test/README.md index 0eddb72e1f..fee8828d34 100644 --- a/test/README.md +++ b/test/README.md @@ -331,6 +331,7 @@ Use the `-v` option for verbose output. | Lint test | Dependency | |-----------|:----------:| | [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) +| [`lint-python.py`](lint/lint-python.py) | [lief](https://github.com/lief-project/LIEF) | [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) | [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) | [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index fa1bb6506a..586722aa65 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -40,7 +40,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - self.nodes[0].wait_until_stopped(timeout=5) + self.nodes[0].wait_until_stopped(timeout=5, expect_error=True) self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index 4f8541a5d7..2ffb182946 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -52,6 +52,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): self._test_use_index_option() self._test_reorg_index() self._test_index_rejects_hash_serialized() + self._test_init_index_after_reorg() def block_sanity_check(self, block_info): block_subsidy = 50 @@ -60,6 +61,9 @@ class CoinStatsIndexTest(BitcoinTestFramework): block_info['new_outputs_ex_coinbase'] + block_info['coinbase'] + block_info['unspendable'] ) + def sync_index_node(self): + self.wait_until(lambda: self.nodes[1].getindexinfo()['coinstatsindex']['synced'] is True) + def _test_coin_stats_index(self): node = self.nodes[0] index_node = self.nodes[1] @@ -145,14 +149,14 @@ class CoinStatsIndexTest(BitcoinTestFramework): self.block_sanity_check(res5['block_info']) # Generate and send a normal tx with two outputs - tx1_txid, tx1_vout = self.wallet.send_to( + tx1 = self.wallet.send_to( from_node=node, scriptPubKey=self.wallet.get_scriptPubKey(), amount=21 * COIN, ) # Find the right position of the 21 BTC output - tx1_out_21 = self.wallet.get_utxo(txid=tx1_txid, vout=tx1_vout) + tx1_out_21 = self.wallet.get_utxo(txid=tx1["txid"], vout=tx1["sent_vout"]) # Generate and send another tx with an OP_RETURN output (which is unspendable) tx2 = self.wallet.create_self_transfer(utxo_to_spend=tx1_out_21)['tx'] @@ -227,18 +231,16 @@ class CoinStatsIndexTest(BitcoinTestFramework): self.log.info("Test that the index works with -reindex") self.restart_node(1, extra_args=["-coinstatsindex", "-reindex"]) + self.sync_index_node() res11 = index_node.gettxoutsetinfo('muhash') assert_equal(res11, res10) - self.log.info("Test that -reindex-chainstate is disallowed with coinstatsindex") + self.log.info("Test that the index works with -reindex-chainstate") - self.stop_node(1) - self.nodes[1].assert_start_raises_init_error( - expected_msg='Error: -reindex-chainstate option is not compatible with -coinstatsindex. ' - 'Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', - extra_args=['-coinstatsindex', '-reindex-chainstate'], - ) - self.restart_node(1, extra_args=["-coinstatsindex"]) + self.restart_node(1, extra_args=["-coinstatsindex", "-reindex-chainstate"]) + self.sync_index_node() + res12 = index_node.gettxoutsetinfo('muhash') + assert_equal(res12, res10) def _test_use_index_option(self): self.log.info("Test use_index option for nodes running the index") @@ -257,6 +259,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): index_node = self.nodes[1] reorg_blocks = self.generatetoaddress(index_node, 2, getnewdestination()[2]) reorg_block = reorg_blocks[1] + self.sync_index_node() res_invalid = index_node.gettxoutsetinfo('muhash') index_node.invalidateblock(reorg_blocks[0]) assert_equal(index_node.gettxoutsetinfo('muhash')['height'], 110) @@ -296,6 +299,26 @@ class CoinStatsIndexTest(BitcoinTestFramework): for use_index in {True, False, None}: assert_raises_rpc_error(-8, msg, self.nodes[1].gettxoutsetinfo, hash_type='hash_serialized_2', hash_or_height=111, use_index=use_index) + def _test_init_index_after_reorg(self): + self.log.info("Test a reorg while the index is deactivated") + index_node = self.nodes[1] + block = self.nodes[0].getbestblockhash() + self.generate(index_node, 2, sync_fun=self.no_op) + self.sync_index_node() + + # Restart without index + self.restart_node(1, extra_args=[]) + self.connect_nodes(0, 1) + index_node.invalidateblock(block) + self.generatetoaddress(index_node, 5, getnewdestination()[2]) + res = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False) + + # Restart with index that still has its best block on the old chain + self.restart_node(1, extra_args=self.extra_args[1]) + self.sync_index_node() + res1 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=True) + assert_equal(res["muhash"], res1["muhash"]) + if __name__ == '__main__': CoinStatsIndexTest().main() diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index f9730b48c5..2257605870 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -5,9 +5,14 @@ """Test various command line arguments and configuration file parameters.""" import os +import pathlib +import re +import sys +import tempfile import time from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_node import ErrorMatch from test_framework import util @@ -74,7 +79,7 @@ class ConfArgsTest(BitcoinTestFramework): util.write_config(main_conf_file_path, n=0, chain='', extra_config=f'includeconf={inc_conf_file_path}\n') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('acceptnonstdtxn=1\n') - self.nodes[0].assert_start_raises_init_error(extra_args=[f"-conf={main_conf_file_path}"], expected_msg='Error: acceptnonstdtxn is not currently supported for main chain') + self.nodes[0].assert_start_raises_init_error(extra_args=[f"-conf={main_conf_file_path}", "-allowignoredconf"], expected_msg='Error: acceptnonstdtxn is not currently supported for main chain') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('nono\n') @@ -108,6 +113,41 @@ class ConfArgsTest(BitcoinTestFramework): with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: conf.write('') # clear + def test_config_file_log(self): + # Disable this test for windows currently because trying to override + # the default datadir through the environment does not seem to work. + if sys.platform == "win32": + return + + self.log.info('Test that correct configuration path is changed when configuration file changes the datadir') + + # Create a temporary directory that will be treated as the default data + # directory by bitcoind. + env, default_datadir = util.get_temp_default_datadir(pathlib.Path(self.options.tmpdir, "test_config_file_log")) + default_datadir.mkdir(parents=True) + + # Write a bitcoin.conf file in the default data directory containing a + # datadir= line pointing at the node datadir. + node = self.nodes[0] + conf_text = pathlib.Path(node.bitcoinconf).read_text() + conf_path = default_datadir / "bitcoin.conf" + conf_path.write_text(f"datadir={node.datadir}\n{conf_text}") + + # Drop the node -datadir= argument during this test, because if it is + # specified it would take precedence over the datadir setting in the + # config file. + node_args = node.args + node.args = [arg for arg in node.args if not arg.startswith("-datadir=")] + + # Check that correct configuration file path is actually logged + # (conf_path, not node.bitcoinconf) + with self.nodes[0].assert_debug_log(expected_msgs=[f"Config file: {conf_path}"]): + self.start_node(0, ["-allowignoredconf"], env=env) + self.stop_node(0) + + # Restore node arguments after the test + node.args = node_args + def test_invalid_command_line_options(self): self.nodes[0].assert_start_raises_init_error( expected_msg='Error: Error parsing command line arguments: Can not set -proxy with no value. Please specify value with -proxy=value.', @@ -282,6 +322,55 @@ class ConfArgsTest(BitcoinTestFramework): unexpected_msgs=seednode_ignored): self.restart_node(0, extra_args=[connect_arg, '-seednode=fakeaddress2']) + def test_ignored_conf(self): + self.log.info('Test error is triggered when the datadir in use contains a bitcoin.conf file that would be ignored ' + 'because a conflicting -conf file argument is passed.') + node = self.nodes[0] + with tempfile.NamedTemporaryFile(dir=self.options.tmpdir, mode="wt", delete=False) as temp_conf: + temp_conf.write(f"datadir={node.datadir}\n") + node.assert_start_raises_init_error([f"-conf={temp_conf.name}"], re.escape( + f'Error: Data directory "{node.datadir}" contains a "bitcoin.conf" file which is ignored, because a ' + f'different configuration file "{temp_conf.name}" from command line argument "-conf={temp_conf.name}" ' + f'is being used instead.') + r"[\s\S]*", match=ErrorMatch.FULL_REGEX) + + # Test that passing a redundant -conf command line argument pointing to + # the same bitcoin.conf that would be loaded anyway does not trigger an + # error. + self.start_node(0, [f'-conf={node.datadir}/bitcoin.conf']) + self.stop_node(0) + + def test_ignored_default_conf(self): + # Disable this test for windows currently because trying to override + # the default datadir through the environment does not seem to work. + if sys.platform == "win32": + return + + self.log.info('Test error is triggered when bitcoin.conf in the default data directory sets another datadir ' + 'and it contains a different bitcoin.conf file that would be ignored') + + # Create a temporary directory that will be treated as the default data + # directory by bitcoind. + env, default_datadir = util.get_temp_default_datadir(pathlib.Path(self.options.tmpdir, "home")) + default_datadir.mkdir(parents=True) + + # Write a bitcoin.conf file in the default data directory containing a + # datadir= line pointing at the node datadir. This will trigger a + # startup error because the node datadir contains a different + # bitcoin.conf that would be ignored. + node = self.nodes[0] + (default_datadir / "bitcoin.conf").write_text(f"datadir={node.datadir}\n") + + # Drop the node -datadir= argument during this test, because if it is + # specified it would take precedence over the datadir setting in the + # config file. + node_args = node.args + node.args = [arg for arg in node.args if not arg.startswith("-datadir=")] + node.assert_start_raises_init_error([], re.escape( + f'Error: Data directory "{node.datadir}" contains a "bitcoin.conf" file which is ignored, because a ' + f'different configuration file "{default_datadir}/bitcoin.conf" from data directory "{default_datadir}" ' + f'is being used instead.') + r"[\s\S]*", env=env, match=ErrorMatch.FULL_REGEX) + node.args = node_args + def run_test(self): self.test_log_buffer() self.test_args_log() @@ -290,7 +379,10 @@ class ConfArgsTest(BitcoinTestFramework): self.test_connect_with_seednode() self.test_config_file_parser() + self.test_config_file_log() self.test_invalid_command_line_options() + self.test_ignored_conf() + self.test_ignored_default_conf() # Remove the -datadir argument so it doesn't override the config file self.nodes[0].args = [arg for arg in self.nodes[0].args if not arg.startswith("-datadir")] diff --git a/test/functional/feature_fastprune.py b/test/functional/feature_fastprune.py index 825de63e3d..c913c4f93a 100755 --- a/test/functional/feature_fastprune.py +++ b/test/functional/feature_fastprune.py @@ -7,11 +7,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal ) -from test_framework.blocktools import ( - create_block, - create_coinbase, - add_witness_commitment -) from test_framework.wallet import MiniWallet @@ -24,18 +19,10 @@ class FeatureFastpruneTest(BitcoinTestFramework): self.log.info("ensure that large blocks don't crash or freeze in -fastprune") wallet = MiniWallet(self.nodes[0]) tx = wallet.create_self_transfer()['tx'] - annex = [0x50] - for _ in range(0x10000): - annex.append(0xff) - tx.wit.vtxinwit[0].scriptWitness.stack.append(bytes(annex)) - tip = int(self.nodes[0].getbestblockhash(), 16) - time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1 - height = self.nodes[0].getblockcount() + 1 - block = create_block(hashprev=tip, ntime=time, txlist=[tx], coinbase=create_coinbase(height=height)) - add_witness_commitment(block) - block.solve() - self.nodes[0].submitblock(block.serialize().hex()) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + annex = b"\x50" + b"\xff" * 0x10000 + tx.wit.vtxinwit[0].scriptWitness.stack.append(annex) + self.generateblock(self.nodes[0], output="raw(55)", transactions=[tx.serialize().hex()]) + assert_equal(self.nodes[0].getblockcount(), 201) if __name__ == '__main__': diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 70a802dc58..7af67730bd 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -45,6 +45,13 @@ class InitStressTest(BitcoinTestFramework): node.process.terminate() node.process.wait() + def start_expecting_error(err_fragment): + node.assert_start_raises_init_error( + extra_args=['-txindex=1', '-blockfilterindex=1', '-coinstatsindex=1'], + expected_msg=err_fragment, + match=ErrorMatch.PARTIAL_REGEX, + ) + def check_clean_start(): """Ensure that node restarts successfully after various interrupts.""" node.start() @@ -87,36 +94,27 @@ class InitStressTest(BitcoinTestFramework): self.log.info("Test startup errors after removing certain essential files") - files_to_disturb = { + files_to_delete = { 'blocks/index/*.ldb': 'Error opening block database.', 'chainstate/*.ldb': 'Error opening block database.', 'blocks/blk*.dat': 'Error loading block database.', } - for file_patt, err_fragment in files_to_disturb.items(): + files_to_perturb = { + 'blocks/index/*.ldb': 'Error opening block database.', + 'chainstate/*.ldb': 'Error opening block database.', + 'blocks/blk*.dat': 'Error opening block database.', + } + + for file_patt, err_fragment in files_to_delete.items(): target_files = list(node.chain_path.glob(file_patt)) for target_file in target_files: - self.log.info(f"Tweaking file to ensure failure {target_file}") + self.log.info(f"Deleting file to ensure failure {target_file}") bak_path = str(target_file) + ".bak" target_file.rename(bak_path) - # TODO: at some point, we should test perturbing the files instead of removing - # them, e.g. - # - # contents = target_file.read_bytes() - # tweaked_contents = bytearray(contents) - # tweaked_contents[50:250] = b'1' * 200 - # target_file.write_bytes(bytes(tweaked_contents)) - # - # At the moment I can't get this to work (bitcoind loads successfully?) so - # investigate doing this later. - - node.assert_start_raises_init_error( - extra_args=['-txindex=1', '-blockfilterindex=1', '-coinstatsindex=1'], - expected_msg=err_fragment, - match=ErrorMatch.PARTIAL_REGEX, - ) + start_expecting_error(err_fragment) for target_file in target_files: bak_path = str(target_file) + ".bak" @@ -126,6 +124,18 @@ class InitStressTest(BitcoinTestFramework): check_clean_start() self.stop_node(0) + for file_patt, err_fragment in files_to_perturb.items(): + target_files = list(node.chain_path.glob(file_patt)) + + for target_file in target_files: + self.log.info(f"Perturbing file to ensure failure {target_file}") + with open(target_file, "rb") as tf_read, open(target_file, "wb") as tf_write: + contents = tf_read.read() + tweaked_contents = bytearray(contents) + tweaked_contents[50:250] = b'1' * 200 + tf_write.write(bytes(tweaked_contents)) + + start_expecting_error(err_fragment) if __name__ == '__main__': InitStressTest().main() diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 947d2e8273..c5eeaf66e0 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -93,7 +93,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): confirmed - txout created will be confirmed in the blockchain; unconfirmed otherwise. """ - txid, n = self.wallet.send_to(from_node=node, scriptPubKey=scriptPubKey or self.wallet.get_scriptPubKey(), amount=amount) + tx = self.wallet.send_to(from_node=node, scriptPubKey=scriptPubKey or self.wallet.get_scriptPubKey(), amount=amount) if confirmed: mempool_size = len(node.getrawmempool()) @@ -105,7 +105,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert new_size < mempool_size mempool_size = new_size - return self.wallet.get_utxo(txid=txid, vout=n) + return self.wallet.get_utxo(txid=tx["txid"], vout=tx["sent_vout"]) def test_simple_doublespend(self): """Simple doublespend""" diff --git a/test/functional/feature_signet.py b/test/functional/feature_signet.py index b41fe378af..a90a2a8e5e 100755 --- a/test/functional/feature_signet.py +++ b/test/functional/feature_signet.py @@ -76,6 +76,9 @@ class SignetBasicTest(BitcoinTestFramework): self.log.info("test that signet logs the network magic on node start") with self.nodes[0].assert_debug_log(["Signet derived magic (message start)"]): self.restart_node(0) + self.stop_node(0) + self.nodes[0].assert_start_raises_init_error(extra_args=["-signetchallenge=abc"], expected_msg="Error: -signetchallenge must be hex, not 'abc'.") + self.nodes[0].assert_start_raises_init_error(extra_args=["-signetchallenge=abc"] * 2, expected_msg="Error: -signetchallenge cannot be multiple values.") if __name__ == '__main__': diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 5017f77d18..1ba8f60d99 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -96,7 +96,7 @@ class RESTTest (BitcoinTestFramework): self.wallet = MiniWallet(self.nodes[0]) self.log.info("Broadcast test transaction and sync nodes") - txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=int(0.1 * COIN)) + txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=int(0.1 * COIN))["txid"] self.sync_all() self.log.info("Test the /tx URI") @@ -173,7 +173,7 @@ class RESTTest (BitcoinTestFramework): # found with or without /checkmempool. # do a tx and don't sync - txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=int(0.1 * COIN)) + txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=int(0.1 * COIN))["txid"] json_obj = self.test_rest_request(f"/tx/{txid}") # get the spent output to later check for utxo (should be spent by then) spent = (json_obj['vin'][0]['txid'], json_obj['vin'][0]['vout']) @@ -421,6 +421,10 @@ class RESTTest (BitcoinTestFramework): deployment_info = self.nodes[0].getdeploymentinfo() assert_equal(deployment_info, self.test_rest_request('/deploymentinfo')) + previous_bb_hash = self.nodes[0].getblockhash(self.nodes[0].getblockcount() - 1) + deployment_info = self.nodes[0].getdeploymentinfo(previous_bb_hash) + assert_equal(deployment_info, self.test_rest_request(f"/deploymentinfo/{previous_bb_hash}")) + non_existing_blockhash = '42759cde25462784395a337460bde75f58e73d3f08bd31fdc3507cbac856a2c4' resp = self.test_rest_request(f'/deploymentinfo/{non_existing_blockhash}', ret_type=RetType.OBJ, status=400) assert_equal(resp.read().decode('utf-8').rstrip(), "Block not found") diff --git a/test/functional/interface_usdt_mempool.py b/test/functional/interface_usdt_mempool.py index ec2f9e4e77..7f088a3ca8 100755 --- a/test/functional/interface_usdt_mempool.py +++ b/test/functional/interface_usdt_mempool.py @@ -35,7 +35,7 @@ MEMPOOL_TRACEPOINTS_PROGRAM = """ struct added_event { u8 hash[HASH_LENGTH]; - u64 vsize; + s32 vsize; s64 fee; }; @@ -43,7 +43,7 @@ struct removed_event { u8 hash[HASH_LENGTH]; char reason[MAX_REMOVAL_REASON_LENGTH]; - u64 vsize; + s32 vsize; s64 fee; u64 entry_time; }; @@ -57,11 +57,11 @@ struct rejected_event struct replaced_event { u8 replaced_hash[HASH_LENGTH]; - u64 replaced_vsize; + s32 replaced_vsize; s64 replaced_fee; u64 replaced_entry_time; u8 replacement_hash[HASH_LENGTH]; - u64 replacement_vsize; + s32 replacement_vsize; s64 replacement_fee; }; @@ -173,6 +173,7 @@ class MempoolTracepointTest(BitcoinTestFramework): self.log.info("Ensuring mempool:added event was handled successfully...") assert_equal(EXPECTED_ADDED_EVENTS, handled_added_events) + self.generate(self.wallet, 1) def removed_test(self): """Expire a transaction from the mempool and make sure the tracepoint returns @@ -223,6 +224,7 @@ class MempoolTracepointTest(BitcoinTestFramework): self.log.info("Ensuring mempool:removed event was handled successfully...") assert_equal(EXPECTED_REMOVED_EVENTS, handled_removed_events) + self.generate(self.wallet, 1) def replaced_test(self): """Replace one and two transactions in the mempool and make sure the tracepoint @@ -280,6 +282,7 @@ class MempoolTracepointTest(BitcoinTestFramework): self.log.info("Ensuring mempool:replaced event was handled successfully...") assert_equal(EXPECTED_REPLACED_EVENTS, handled_replaced_events) + self.generate(self.wallet, 1) def rejected_test(self): """Create an invalid transaction and make sure the tracepoint returns @@ -321,6 +324,7 @@ class MempoolTracepointTest(BitcoinTestFramework): self.log.info("Ensuring mempool:rejected event was handled successfully...") assert_equal(EXPECTED_REJECTED_EVENTS, handled_rejected_events) + self.generate(self.wallet, 1) def run_test(self): """Tests the mempool:added, mempool:removed, mempool:replaced, diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index 23785b1e8a..6774db7c5f 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -363,7 +363,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) self.log.info("stop the node to flush the UTXO cache") - UTXOS_IN_CACHE = 2 # might need to be changed if the eariler tests are modified + UTXOS_IN_CACHE = 2 # might need to be changed if the earlier tests are modified # A node shutdown causes two flushes. One that flushes UTXOS_IN_CACHE # UTXOs and one that flushes 0 UTXOs. Normally the 0-UTXO-flush is the # second flush, however it can happen that the order changes. diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 362b407062..737a8d0a2e 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -350,7 +350,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A tiny transaction(in non-witness bytes) that is disallowed') tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(int(seed_tx[0], 16), seed_tx[1]), b"", SEQUENCE_FINAL)) + tx.vin.append(CTxIn(COutPoint(int(seed_tx["txid"], 16), seed_tx["sent_vout"]), b"", SEQUENCE_FINAL)) tx.wit.vtxinwit = [CTxInWitness()] tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] tx.vout.append(CTxOut(0, CScript([OP_RETURN] + ([OP_0] * (MIN_PADDING - 2))))) diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index a7bdc49695..7337802aea 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -47,12 +47,12 @@ class MempoolCompatibilityTest(BitcoinTestFramework): # unbroadcasted_tx won't pass old_node's `MemPoolAccept::PreChecks`. self.connect_nodes(0, 1) self.sync_blocks() - self.stop_node(1) self.log.info("Add a transaction to mempool on old node and shutdown") old_tx_hash = new_wallet.send_self_transfer(from_node=old_node)["txid"] assert old_tx_hash in old_node.getrawmempool() self.stop_node(0) + self.stop_node(1) self.log.info("Move mempool.dat from old to new node") old_node_mempool = os.path.join(old_node.datadir, self.chain, 'mempool.dat') diff --git a/test/functional/mempool_expiry.py b/test/functional/mempool_expiry.py index 15a5f765df..18b3a8def4 100755 --- a/test/functional/mempool_expiry.py +++ b/test/functional/mempool_expiry.py @@ -12,7 +12,10 @@ definable expiry timeout via the '-mempoolexpiry=<n>' command line argument from datetime import timedelta -from test_framework.messages import DEFAULT_MEMPOOL_EXPIRY_HOURS +from test_framework.messages import ( + COIN, + DEFAULT_MEMPOOL_EXPIRY_HOURS, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -37,6 +40,10 @@ class MempoolExpiryTest(BitcoinTestFramework): parent_utxo = self.wallet.get_utxo(txid=parent_txid) independent_utxo = self.wallet.get_utxo() + # Add prioritisation to this transaction to check that it persists after the expiry + node.prioritisetransaction(parent_txid, 0, COIN) + assert_equal(node.getprioritisedtransactions()[parent_txid], { "fee_delta" : COIN, "in_mempool" : True}) + # Ensure the transactions we send to trigger the mempool check spend utxos that are independent of # the transactions being tested for expiration. trigger_utxo1 = self.wallet.get_utxo() @@ -79,6 +86,9 @@ class MempoolExpiryTest(BitcoinTestFramework): assert_raises_rpc_error(-5, 'Transaction not in mempool', node.getmempoolentry, parent_txid) + # Prioritisation does not disappear when transaction expires + assert_equal(node.getprioritisedtransactions()[parent_txid], { "fee_delta" : COIN, "in_mempool" : False}) + # The child transaction should be removed from the mempool as well. self.log.info('Test child tx is evicted as well.') assert_raises_rpc_error(-5, 'Transaction not in mempool', diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py index 6143ae83de..81451bf2a5 100755 --- a/test/functional/mempool_package_limits.py +++ b/test/functional/mempool_package_limits.py @@ -46,8 +46,7 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) # Add enough mature utxos to the wallet so that all txs spend confirmed coins. - self.generate(self.wallet, 35) - self.generate(self.nodes[0], COINBASE_MATURITY) + self.generate(self.wallet, COINBASE_MATURITY + 35) self.test_chain_limits() self.test_desc_count_limits() diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 0387282862..95f7939412 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -7,7 +7,6 @@ from decimal import Decimal from test_framework.messages import ( - COIN, DEFAULT_ANCESTOR_LIMIT, DEFAULT_DESCENDANT_LIMIT, ) @@ -26,9 +25,6 @@ assert CUSTOM_DESCENDANT_LIMIT >= CUSTOM_ANCESTOR_LIMIT class MempoolPackagesTest(BitcoinTestFramework): - def add_options(self, parser): - self.add_wallet_options(parser) - def set_test_params(self): self.num_nodes = 2 self.extra_args = [ @@ -47,10 +43,6 @@ class MempoolPackagesTest(BitcoinTestFramework): self.wallet = MiniWallet(self.nodes[0]) self.wallet.rescan_utxos() - if self.is_specified_wallet_compiled(): - self.nodes[0].createwallet("watch_wallet", disable_private_keys=True) - self.nodes[0].importaddress(self.wallet.get_address()) - peer_inv_store = self.nodes[0].add_p2p_connection(P2PTxInvStore()) # keep track of invs # DEFAULT_ANCESTOR_LIMIT transactions off a confirmed tx should be fine @@ -63,13 +55,6 @@ class MempoolPackagesTest(BitcoinTestFramework): ancestor_vsize += t["tx"].get_vsize() ancestor_fees += t["fee"] self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=t["hex"]) - # Check that listunspent ancestor{count, size, fees} yield the correct results - if self.is_specified_wallet_compiled(): - wallet_unspent = self.nodes[0].listunspent(minconf=0) - this_unspent = next(utxo_info for utxo_info in wallet_unspent if utxo_info["txid"] == t["txid"]) - assert_equal(this_unspent['ancestorcount'], i + 1) - assert_equal(this_unspent['ancestorsize'], ancestor_vsize) - assert_equal(this_unspent['ancestorfees'], ancestor_fees * COIN) # Wait until mempool transactions have passed initial broadcast (sent inv and received getdata) # Otherwise, getrawmempool may be inconsistent with getmempoolentry if unbroadcast changes in between diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index f818801136..8f74d9de20 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -191,6 +191,7 @@ class MempoolPersistTest(BitcoinTestFramework): def test_persist_unbroadcast(self): node0 = self.nodes[0] self.start_node(0) + self.start_node(2) # clear out mempool self.generate(node0, 1, sync_fun=self.no_op) diff --git a/test/functional/mempool_sigoplimit.py b/test/functional/mempool_sigoplimit.py index b178b9feda..962b2b19bd 100755 --- a/test/functional/mempool_sigoplimit.py +++ b/test/functional/mempool_sigoplimit.py @@ -49,7 +49,7 @@ class BytesPerSigOpTest(BitcoinTestFramework): """Create a 1-input-1-output P2WSH spending transaction with only the witness script in the witness stack and the given output script.""" # create P2WSH address and fund it via MiniWallet first - txid, vout = self.wallet.send_to( + fund = self.wallet.send_to( from_node=self.nodes[0], scriptPubKey=script_to_p2wsh_script(witness_script), amount=1000000, @@ -57,7 +57,7 @@ class BytesPerSigOpTest(BitcoinTestFramework): # create spending transaction tx = CTransaction() - tx.vin = [CTxIn(COutPoint(int(txid, 16), vout))] + tx.vin = [CTxIn(COutPoint(int(fund["txid"], 16), fund["sent_vout"]))] tx.wit.vtxinwit = [CTxInWitness()] tx.wit.vtxinwit[0].scriptWitness.stack = [bytes(witness_script)] tx.vout = [CTxOut(500000, output_script)] diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 332099516c..aabf06ee53 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -145,6 +145,7 @@ class MiningTest(BitcoinTestFramework): assert_template(node, bad_block, 'bad-cb-missing') self.log.info("submitblock: Test invalid coinbase transaction") + assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, CBlock().serialize().hex()) assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, bad_block.serialize().hex()) self.log.info("getblocktemplate: Test truncated final transaction") diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index a4481c15a0..099c0e418c 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -30,6 +30,33 @@ class PrioritiseTransactionTest(BitcoinTestFramework): ]] * self.num_nodes self.supports_cli = False + def clear_prioritisation(self, node): + for txid, info in node.getprioritisedtransactions().items(): + delta = info["fee_delta"] + node.prioritisetransaction(txid, 0, -delta) + assert_equal(node.getprioritisedtransactions(), {}) + + def test_replacement(self): + self.log.info("Test tx prioritisation stays after a tx is replaced") + conflicting_input = self.wallet.get_utxo() + tx_replacee = self.wallet.create_self_transfer(utxo_to_spend=conflicting_input, fee_rate=Decimal("0.0001")) + tx_replacement = self.wallet.create_self_transfer(utxo_to_spend=conflicting_input, fee_rate=Decimal("0.005")) + # Add 1 satoshi fee delta to replacee + self.nodes[0].prioritisetransaction(tx_replacee["txid"], 0, 100) + assert_equal(self.nodes[0].getprioritisedtransactions(), { tx_replacee["txid"] : { "fee_delta" : 100, "in_mempool" : False}}) + self.nodes[0].sendrawtransaction(tx_replacee["hex"]) + assert_equal(self.nodes[0].getprioritisedtransactions(), { tx_replacee["txid"] : { "fee_delta" : 100, "in_mempool" : True}}) + self.nodes[0].sendrawtransaction(tx_replacement["hex"]) + assert tx_replacee["txid"] not in self.nodes[0].getrawmempool() + assert_equal(self.nodes[0].getprioritisedtransactions(), { tx_replacee["txid"] : { "fee_delta" : 100, "in_mempool" : False}}) + + # PrioritiseTransaction is additive + self.nodes[0].prioritisetransaction(tx_replacee["txid"], 0, COIN) + self.nodes[0].sendrawtransaction(tx_replacee["hex"]) + assert_equal(self.nodes[0].getprioritisedtransactions(), { tx_replacee["txid"] : { "fee_delta" : COIN + 100, "in_mempool" : True}}) + self.generate(self.nodes[0], 1) + assert_equal(self.nodes[0].getprioritisedtransactions(), {}) + def test_diamond(self): self.log.info("Test diamond-shape package with priority") mock_time = int(time.time()) @@ -84,6 +111,13 @@ class PrioritiseTransactionTest(BitcoinTestFramework): raw_after = self.nodes[0].getrawmempool(verbose=True) assert_equal(raw_before[txid_a], raw_after[txid_a]) assert_equal(raw_before, raw_after) + prioritisation_map_in_mempool = self.nodes[0].getprioritisedtransactions() + assert_equal(prioritisation_map_in_mempool[txid_b], {"fee_delta" : fee_delta_b*COIN, "in_mempool" : True}) + assert_equal(prioritisation_map_in_mempool[txid_c], {"fee_delta" : (fee_delta_c_1 + fee_delta_c_2)*COIN, "in_mempool" : True}) + # Clear prioritisation, otherwise the transactions' fee deltas are persisted to mempool.dat and loaded again when the node + # is restarted at the end of this subtest. Deltas are removed when a transaction is mined, but only at that time. We do + # not check whether mapDeltas transactions were mined when loading from mempool.dat. + self.clear_prioritisation(node=self.nodes[0]) self.log.info("Test priority while txs are not in mempool") self.restart_node(0, extra_args=["-nopersistmempool"]) @@ -92,17 +126,26 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_1 * COIN)) self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_2 * COIN)) + prioritisation_map_not_in_mempool = self.nodes[0].getprioritisedtransactions() + assert_equal(prioritisation_map_not_in_mempool[txid_b], {"fee_delta" : fee_delta_b*COIN, "in_mempool" : False}) + assert_equal(prioritisation_map_not_in_mempool[txid_c], {"fee_delta" : (fee_delta_c_1 + fee_delta_c_2)*COIN, "in_mempool" : False}) for t in [tx_o_a["hex"], tx_o_b["hex"], tx_o_c["hex"], tx_o_d["hex"]]: self.nodes[0].sendrawtransaction(t) raw_after = self.nodes[0].getrawmempool(verbose=True) assert_equal(raw_before[txid_a], raw_after[txid_a]) assert_equal(raw_before, raw_after) + prioritisation_map_in_mempool = self.nodes[0].getprioritisedtransactions() + assert_equal(prioritisation_map_in_mempool[txid_b], {"fee_delta" : fee_delta_b*COIN, "in_mempool" : True}) + assert_equal(prioritisation_map_in_mempool[txid_c], {"fee_delta" : (fee_delta_c_1 + fee_delta_c_2)*COIN, "in_mempool" : True}) # Clear mempool self.generate(self.nodes[0], 1) + # Prioritisation for transactions is automatically deleted after they are mined. + assert_equal(self.nodes[0].getprioritisedtransactions(), {}) # Use default extra_args self.restart_node(0) + assert_equal(self.nodes[0].getprioritisedtransactions(), {}) def run_test(self): self.wallet = MiniWallet(self.nodes[0]) @@ -115,6 +158,10 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Test `prioritisetransaction` invalid extra parameters assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction, '', 0, 0, 0) + # Test `getprioritisedtransactions` invalid parameters + assert_raises_rpc_error(-1, "getprioritisedtransactions", + self.nodes[0].getprioritisedtransactions, True) + # Test `prioritisetransaction` invalid `txid` assert_raises_rpc_error(-8, "txid must be of length 64 (not 3, for 'foo')", self.nodes[0].prioritisetransaction, txid='foo', fee_delta=0) assert_raises_rpc_error(-8, "txid must be hexadecimal string (not 'Zd1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000')", self.nodes[0].prioritisetransaction, txid='Zd1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000', fee_delta=0) @@ -127,6 +174,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Test `prioritisetransaction` invalid `fee_delta` assert_raises_rpc_error(-3, "JSON value of type string is not of expected type number", self.nodes[0].prioritisetransaction, txid=txid, fee_delta='foo') + self.test_replacement() self.test_diamond() self.txouts = gen_return_txouts() @@ -165,9 +213,18 @@ class PrioritiseTransactionTest(BitcoinTestFramework): sizes[i] += mempool[j]['vsize'] assert sizes[i] > MAX_BLOCK_WEIGHT // 4 # Fail => raise utxo_count + assert_equal(self.nodes[0].getprioritisedtransactions(), {}) # add a fee delta to something in the cheapest bucket and make sure it gets mined # also check that a different entry in the cheapest bucket is NOT mined self.nodes[0].prioritisetransaction(txid=txids[0][0], fee_delta=int(3*base_fee*COIN)) + assert_equal(self.nodes[0].getprioritisedtransactions(), {txids[0][0] : { "fee_delta" : 3*base_fee*COIN, "in_mempool" : True}}) + + # Priority disappears when prioritisetransaction is called with an inverse value... + self.nodes[0].prioritisetransaction(txid=txids[0][0], fee_delta=int(-3*base_fee*COIN)) + assert txids[0][0] not in self.nodes[0].getprioritisedtransactions() + # ... and reappears when prioritisetransaction is called again. + self.nodes[0].prioritisetransaction(txid=txids[0][0], fee_delta=int(3*base_fee*COIN)) + assert txids[0][0] in self.nodes[0].getprioritisedtransactions() self.generate(self.nodes[0], 1) @@ -187,6 +244,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Add a prioritisation before a tx is in the mempool (de-prioritising a # high-fee transaction so that it's now low fee). self.nodes[0].prioritisetransaction(txid=high_fee_tx, fee_delta=-int(2*base_fee*COIN)) + assert_equal(self.nodes[0].getprioritisedtransactions()[high_fee_tx], { "fee_delta" : -2*base_fee*COIN, "in_mempool" : False}) # Add everything back to mempool self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) @@ -206,6 +264,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): mempool = self.nodes[0].getrawmempool() self.log.info("Assert that de-prioritised transaction is still in mempool") assert high_fee_tx in mempool + assert_equal(self.nodes[0].getprioritisedtransactions()[high_fee_tx], { "fee_delta" : -2*base_fee*COIN, "in_mempool" : True}) for x in txids[2]: if (x != high_fee_tx): assert x not in mempool @@ -223,10 +282,12 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # to be the minimum for a 1000-byte transaction and check that it is # accepted. self.nodes[0].prioritisetransaction(txid=tx_id, fee_delta=int(self.relayfee*COIN)) + assert_equal(self.nodes[0].getprioritisedtransactions()[tx_id], { "fee_delta" : self.relayfee*COIN, "in_mempool" : False}) self.log.info("Assert that prioritised free transaction is accepted to mempool") assert_equal(self.nodes[0].sendrawtransaction(tx_hex), tx_id) assert tx_id in self.nodes[0].getrawmempool() + assert_equal(self.nodes[0].getprioritisedtransactions()[tx_id], { "fee_delta" : self.relayfee*COIN, "in_mempool" : True}) # Test that calling prioritisetransaction is sufficient to trigger # getblocktemplate to (eventually) return a new block. @@ -234,6 +295,10 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.nodes[0].setmocktime(mock_time) template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) self.nodes[0].prioritisetransaction(txid=tx_id, fee_delta=-int(self.relayfee*COIN)) + + # Calling prioritisetransaction with the inverse amount should delete its prioritisation entry + assert tx_id not in self.nodes[0].getprioritisedtransactions() + self.nodes[0].setmocktime(mock_time+10) new_template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index 2da9037a69..e4908735c9 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -255,13 +255,6 @@ class CompactFiltersTest(BitcoinTestFramework): msg = "Error: Unknown -blockfilterindex value abc." self.nodes[0].assert_start_raises_init_error(expected_msg=msg) - self.log.info("Test -blockfilterindex with -reindex-chainstate raises an error") - self.nodes[0].assert_start_raises_init_error( - expected_msg='Error: -reindex-chainstate option is not compatible with -blockfilterindex. ' - 'Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', - extra_args=['-blockfilterindex', '-reindex-chainstate'], - ) - def compute_last_header(prev_header, hashes): """Compute the last filter header from a starting header and a sequence of filter hashes.""" header = ser_uint256(prev_header) diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 23eeea50bc..d6c06fdeed 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -105,6 +105,10 @@ class TestP2PConn(P2PInterface): self.last_message.pop("headers", None) self.last_message.pop("cmpctblock", None) + def clear_getblocktxn(self): + with p2p_lock: + self.last_message.pop("getblocktxn", None) + def get_headers(self, locator, hashstop): msg = msg_getheaders() msg.locator.vHave = locator @@ -745,7 +749,7 @@ class CompactBlocksTest(BitcoinTestFramework): peer.get_headers(locator=[int(tip, 16)], hashstop=0) peer.send_and_ping(msg_sendcmpct(announce=True, version=2)) - def test_compactblock_reconstruction_multiple_peers(self, stalling_peer, delivery_peer): + def test_compactblock_reconstruction_stalling_peer(self, stalling_peer, delivery_peer): node = self.nodes[0] assert len(self.utxos) @@ -823,12 +827,85 @@ class CompactBlocksTest(BitcoinTestFramework): hb_test_node.send_and_ping(msg_sendcmpct(announce=False, version=2)) assert_highbandwidth_states(self.nodes[0], hb_to=True, hb_from=False) + def test_compactblock_reconstruction_parallel_reconstruction(self, stalling_peer, delivery_peer, inbound_peer, outbound_peer): + """ All p2p connections are inbound except outbound_peer. We test that ultimate parallel slot + can only be taken by an outbound node unless prior attempts were done by an outbound + """ + node = self.nodes[0] + assert len(self.utxos) + + def announce_cmpct_block(node, peer, txn_count): + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(node, utxo, txn_count) + + cmpct_block = HeaderAndShortIDs() + cmpct_block.initialize_from_block(block) + msg = msg_cmpctblock(cmpct_block.to_p2p()) + peer.send_and_ping(msg) + with p2p_lock: + assert "getblocktxn" in peer.last_message + return block, cmpct_block + + for name, peer in [("delivery", delivery_peer), ("inbound", inbound_peer), ("outbound", outbound_peer)]: + self.log.info(f"Setting {name} as high bandwidth peer") + block, cmpct_block = announce_cmpct_block(node, peer, 1) + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = block.vtx[1:] + peer.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + peer.clear_getblocktxn() + + # Test the simple parallel download case... + for num_missing in [1, 5, 20]: + + # Remaining low-bandwidth peer is stalling_peer, who announces first + assert_equal([peer['bip152_hb_to'] for peer in node.getpeerinfo()], [False, True, True, True]) + + block, cmpct_block = announce_cmpct_block(node, stalling_peer, num_missing) + + delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) + with p2p_lock: + # The second peer to announce should still get a getblocktxn + assert "getblocktxn" in delivery_peer.last_message + assert int(node.getbestblockhash(), 16) != block.sha256 + + inbound_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) + with p2p_lock: + # The third inbound peer to announce should *not* get a getblocktxn + assert "getblocktxn" not in inbound_peer.last_message + assert int(node.getbestblockhash(), 16) != block.sha256 + + outbound_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) + with p2p_lock: + # The third peer to announce should get a getblocktxn if outbound + assert "getblocktxn" in outbound_peer.last_message + assert int(node.getbestblockhash(), 16) != block.sha256 + + # Second peer completes the compact block first + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = block.vtx[1:] + delivery_peer.send_and_ping(msg) + assert_equal(int(node.getbestblockhash(), 16), block.sha256) + + # Nothing bad should happen if we get a late fill from the first peer... + stalling_peer.send_and_ping(msg) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + delivery_peer.clear_getblocktxn() + inbound_peer.clear_getblocktxn() + outbound_peer.clear_getblocktxn() + + def run_test(self): self.wallet = MiniWallet(self.nodes[0]) # Setup the p2p connections self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) self.additional_segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn()) + self.onemore_inbound_node = self.nodes[0].add_p2p_connection(TestP2PConn()) + self.outbound_node = self.nodes[0].add_outbound_p2p_connection(TestP2PConn(), p2p_idx=3, connection_type="outbound-full-relay") # We will need UTXOs to construct transactions in later tests. self.make_utxos() @@ -838,6 +915,8 @@ class CompactBlocksTest(BitcoinTestFramework): self.log.info("Testing SENDCMPCT p2p message... ") self.test_sendcmpct(self.segwit_node) self.test_sendcmpct(self.additional_segwit_node) + self.test_sendcmpct(self.onemore_inbound_node) + self.test_sendcmpct(self.outbound_node) self.log.info("Testing compactblock construction...") self.test_compactblock_construction(self.segwit_node) @@ -860,8 +939,11 @@ class CompactBlocksTest(BitcoinTestFramework): self.log.info("Testing handling of incorrect blocktxn responses...") self.test_incorrect_blocktxn_response(self.segwit_node) - self.log.info("Testing reconstructing compact blocks from all peers...") - self.test_compactblock_reconstruction_multiple_peers(self.segwit_node, self.additional_segwit_node) + self.log.info("Testing reconstructing compact blocks with a stalling peer...") + self.test_compactblock_reconstruction_stalling_peer(self.segwit_node, self.additional_segwit_node) + + self.log.info("Testing reconstructing compact blocks from multiple peers...") + self.test_compactblock_reconstruction_parallel_reconstruction(stalling_peer=self.segwit_node, inbound_peer=self.onemore_inbound_node, delivery_peer=self.additional_segwit_node, outbound_peer=self.outbound_node) # Test that if we submitblock to node1, we'll get a compact block # announcement to all peers. diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index b3e68ca536..6699cc3528 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -136,7 +136,7 @@ class FilterTest(BitcoinTestFramework): filter_peer = P2PBloomFilter() self.log.debug("Create a tx relevant to the peer before connecting") - txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=filter_peer.watch_script_pubkey, amount=9 * COIN) + txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=filter_peer.watch_script_pubkey, amount=9 * COIN)["txid"] self.log.debug("Send a mempool msg after connecting and check that the tx is received") self.nodes[0].add_p2p_connection(filter_peer) @@ -183,14 +183,14 @@ class FilterTest(BitcoinTestFramework): self.log.info('Check that we receive a tx if the filter matches a mempool tx') filter_peer.merkleblock_received = False - txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=filter_peer.watch_script_pubkey, amount=9 * COIN) + txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=filter_peer.watch_script_pubkey, amount=9 * COIN)["txid"] filter_peer.wait_for_tx(txid) assert not filter_peer.merkleblock_received self.log.info('Check that after deleting filter all txs get relayed again') filter_peer.send_and_ping(msg_filterclear()) for _ in range(5): - txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=7 * COIN) + txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=7 * COIN)["txid"] filter_peer.wait_for_tx(txid) self.log.info('Check that request for filtered blocks is ignored if no filter is set') diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index 6283dd89ac..f53f98e06d 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -2,10 +2,10 @@ # Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test that we don't leak txs to inbound peers that we haven't yet announced to""" +"""Test transaction upload""" -from test_framework.messages import msg_getdata, CInv, MSG_TX -from test_framework.p2p import p2p_lock, P2PDataStore +from test_framework.messages import msg_getdata, CInv, MSG_TX, MSG_WTX +from test_framework.p2p import p2p_lock, P2PDataStore, P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -23,16 +23,67 @@ class P2PLeakTxTest(BitcoinTestFramework): self.num_nodes = 1 def run_test(self): - gen_node = self.nodes[0] # The block and tx generating node - miniwallet = MiniWallet(gen_node) + self.gen_node = self.nodes[0] # The block and tx generating node + self.miniwallet = MiniWallet(self.gen_node) - inbound_peer = self.nodes[0].add_p2p_connection(P2PNode()) # An "attacking" inbound peer + self.test_tx_in_block() + self.test_notfound_on_replaced_tx() + self.test_notfound_on_unannounced_tx() + + def test_tx_in_block(self): + self.log.info("Check that a transaction in the last block is uploaded (beneficial for compact block relay)") + inbound_peer = self.gen_node.add_p2p_connection(P2PNode()) + + self.log.debug("Generate transaction and block") + inbound_peer.last_message.pop("inv", None) + wtxid = self.miniwallet.send_self_transfer(from_node=self.gen_node)["wtxid"] + inbound_peer.wait_until(lambda: "inv" in inbound_peer.last_message and inbound_peer.last_message.get("inv").inv[0].hash == int(wtxid, 16)) + want_tx = msg_getdata(inv=inbound_peer.last_message.get("inv").inv) + self.generate(self.gen_node, 1) + + self.log.debug("Request transaction") + inbound_peer.last_message.pop("tx", None) + inbound_peer.send_and_ping(want_tx) + assert_equal(inbound_peer.last_message.get("tx").tx.getwtxid(), wtxid) + + def test_notfound_on_replaced_tx(self): + self.gen_node.disconnect_p2ps() + inbound_peer = self.gen_node.add_p2p_connection(P2PTxInvStore()) + + self.log.info("Transaction tx_a is broadcast") + tx_a = self.miniwallet.send_self_transfer(from_node=self.gen_node) + inbound_peer.wait_for_broadcast(txns=[tx_a["wtxid"]]) + + tx_b = tx_a["tx"] + tx_b.vout[0].nValue -= 9000 + self.gen_node.sendrawtransaction(tx_b.serialize().hex()) + inbound_peer.wait_until(lambda: "tx" in inbound_peer.last_message and inbound_peer.last_message.get("tx").tx.getwtxid() == tx_b.getwtxid()) + + self.log.info("Re-request of tx_a after replacement is answered with notfound") + req_vec = [ + CInv(t=MSG_TX, h=int(tx_a["txid"], 16)), + CInv(t=MSG_WTX, h=int(tx_a["wtxid"], 16)), + ] + want_tx = msg_getdata() + want_tx.inv = req_vec + with p2p_lock: + inbound_peer.last_message.pop("notfound", None) + inbound_peer.last_message.pop("tx", None) + inbound_peer.send_and_ping(want_tx) + + assert_equal(inbound_peer.last_message.get("notfound").vec, req_vec) + assert "tx" not in inbound_peer.last_message + + def test_notfound_on_unannounced_tx(self): + self.log.info("Check that we don't leak txs to inbound peers that we haven't yet announced to") + self.gen_node.disconnect_p2ps() + inbound_peer = self.gen_node.add_p2p_connection(P2PNode()) # An "attacking" inbound peer MAX_REPEATS = 100 self.log.info("Running test up to {} times.".format(MAX_REPEATS)) for i in range(MAX_REPEATS): self.log.info('Run repeat {}'.format(i + 1)) - txid = miniwallet.send_self_transfer(from_node=gen_node)['wtxid'] + txid = self.miniwallet.send_self_transfer(from_node=self.gen_node)["wtxid"] want_tx = msg_getdata() want_tx.inv.append(CInv(t=MSG_TX, h=int(txid, 16))) diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index bec499107f..279fb01a57 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -195,7 +195,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): wmulti.unloadwallet() spk = address_to_scriptpubkey(madd) - txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=1300) + txid = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=1300)["txid"] tx = node0.getrawtransaction(txid, True) vout = [v["n"] for v in tx["vout"] if madd == v["scriptPubKey"]["address"]] assert len(vout) == 1 diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index 2f093bebff..1ab1023cf1 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -117,9 +117,11 @@ class GetBlockFromPeerTest(BitcoinTestFramework): assert_raises_rpc_error(-1, error_msg, self.nodes[1].getblockfrompeer, blockhash, node1_interface_id) self.log.info("Connect pruned node") - # We need to generate more blocks to be able to prune self.connect_nodes(0, 2) pruned_node = self.nodes[2] + self.sync_blocks([self.nodes[0], pruned_node]) + + # We need to generate more blocks to be able to prune self.generate(self.nodes[0], 400, sync_fun=self.no_op) self.sync_blocks([self.nodes[0], pruned_node]) pruneheight = pruned_node.pruneblockchain(300) diff --git a/test/functional/rpc_getdescriptorinfo.py b/test/functional/rpc_getdescriptorinfo.py index 1b0f411e52..2eb36f260c 100755 --- a/test/functional/rpc_getdescriptorinfo.py +++ b/test/functional/rpc_getdescriptorinfo.py @@ -55,9 +55,9 @@ class DescriptorTest(BitcoinTestFramework): # A P2PK output with the public key of the specified xpub. self.test_desc('pk(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B)', isrange=False, issolvable=True, hasprivatekeys=False) # A P2PKH output with child key *1'/2* of the specified xpub. - self.test_desc("pkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1'/2)", isrange=False, issolvable=True, hasprivatekeys=False) + self.test_desc("pkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1h/2)", isrange=False, issolvable=True, hasprivatekeys=False) # A set of P2PKH outputs, but additionally specifies that the specified xpub is a child of a master with fingerprint `d34db33f`, and derived using path `44'/0'/0'`. - self.test_desc("pkh([d34db33f/44'/0'/0']tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/*)", isrange=True, issolvable=True, hasprivatekeys=False) + self.test_desc("pkh([d34db33f/44h/0h/0h]tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/*)", isrange=True, issolvable=True, hasprivatekeys=False) # A set of *1-of-2* P2WSH multisig outputs where the first multisig key is the *1/0/`i`* child of the first specified xpub and the second multisig key is the *0/0/`i`* child of the second specified xpub, and `i` is any number in a configurable range (`0-1000` by default). self.test_desc("wsh(multi(1,tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/0/*,tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/0/0/*))", isrange=True, issolvable=True, hasprivatekeys=False) diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index 7acc3cbbd5..53c5aa05e5 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -85,8 +85,8 @@ class HelpRpcTest(BitcoinTestFramework): for argname, convert in converts_by_argname.items(): if all(convert) != any(convert): - # Only allow dummy to fail consistency check - assert argname == 'dummy', ('WARNING: conversion mismatch for argument named %s (%s)' % (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) + # Only allow dummy and psbt to fail consistency check + assert argname in ['dummy', "psbt"], ('WARNING: conversion mismatch for argument named %s (%s)' % (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) def test_categories(self): node = self.nodes[0] diff --git a/test/functional/rpc_invalid_address_message.py b/test/functional/rpc_invalid_address_message.py index fd282a9bc1..6759b69dd1 100755 --- a/test/functional/rpc_invalid_address_message.py +++ b/test/functional/rpc_invalid_address_message.py @@ -63,12 +63,12 @@ class InvalidAddressErrorMessageTest(BitcoinTestFramework): def test_validateaddress(self): # Invalid Bech32 - self.check_invalid(BECH32_INVALID_SIZE, 'Invalid Bech32 address data size') + self.check_invalid(BECH32_INVALID_SIZE, "Invalid Bech32 address program size (41 bytes)") self.check_invalid(BECH32_INVALID_PREFIX, 'Invalid or unsupported Segwit (Bech32) or Base58 encoding.') self.check_invalid(BECH32_INVALID_BECH32, 'Version 1+ witness address must use Bech32m checksum') self.check_invalid(BECH32_INVALID_BECH32M, 'Version 0 witness address must use Bech32 checksum') self.check_invalid(BECH32_INVALID_VERSION, 'Invalid Bech32 address witness version') - self.check_invalid(BECH32_INVALID_V0_SIZE, 'Invalid Bech32 v0 address data size') + self.check_invalid(BECH32_INVALID_V0_SIZE, "Invalid Bech32 v0 address program size (21 bytes), per BIP141") self.check_invalid(BECH32_TOO_LONG, 'Bech32 string too long', list(range(90, 108))) self.check_invalid(BECH32_ONE_ERROR, 'Invalid Bech32 checksum', [9]) self.check_invalid(BECH32_TWO_ERRORS, 'Invalid Bech32 checksum', [22, 43]) @@ -105,7 +105,7 @@ class InvalidAddressErrorMessageTest(BitcoinTestFramework): def test_getaddressinfo(self): node = self.nodes[0] - assert_raises_rpc_error(-5, "Invalid Bech32 address data size", node.getaddressinfo, BECH32_INVALID_SIZE) + assert_raises_rpc_error(-5, "Invalid Bech32 address program size (41 bytes)", node.getaddressinfo, BECH32_INVALID_SIZE) assert_raises_rpc_error(-5, "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", node.getaddressinfo, BECH32_INVALID_PREFIX) assert_raises_rpc_error(-5, "Invalid or unsupported Base58-encoded address.", node.getaddressinfo, BASE58_INVALID_PREFIX) assert_raises_rpc_error(-5, "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", node.getaddressinfo, INVALID_ADDRESS) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index ef773463d8..e2fb4673ea 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -42,7 +42,10 @@ from test_framework.util import ( find_vout_for_address, random_bytes, ) -from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet_util import ( + bytes_to_wif, + get_generate_key +) import json import os @@ -373,7 +376,7 @@ class PSBTTest(BitcoinTestFramework): self.log.info("Test various PSBT operations") # partially sign multisig things with node 1 - psbtx = wmulti.walletcreatefundedpsbt(inputs=[{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], outputs={self.nodes[1].getnewaddress():29.99}, options={'changeAddress': self.nodes[1].getrawchangeaddress()})['psbt'] + psbtx = wmulti.walletcreatefundedpsbt(inputs=[{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], outputs={self.nodes[1].getnewaddress():29.99}, changeAddress=self.nodes[1].getrawchangeaddress())['psbt'] walletprocesspsbt_out = self.nodes[1].walletprocesspsbt(psbtx) psbtx = walletprocesspsbt_out['psbt'] assert_equal(walletprocesspsbt_out['complete'], False) @@ -397,17 +400,15 @@ class PSBTTest(BitcoinTestFramework): self.nodes[0].decodepsbt(new_psbt) # Make sure that a non-psbt with signatures cannot be converted - # Error could be either "TX decode failed" (segwit inputs causes parsing to fail) or "Inputs must not have scriptSigs and scriptWitnesses" - # We must set iswitness=True because the serialized transaction has inputs and is therefore a witness transaction signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx['hex']) - assert_raises_rpc_error(-22, "", self.nodes[0].converttopsbt, hexstring=signedtx['hex'], iswitness=True) - assert_raises_rpc_error(-22, "", self.nodes[0].converttopsbt, hexstring=signedtx['hex'], permitsigdata=False, iswitness=True) + assert_raises_rpc_error(-22, "Inputs must not have scriptSigs and scriptWitnesses", + self.nodes[0].converttopsbt, hexstring=signedtx['hex']) # permitsigdata=False by default + assert_raises_rpc_error(-22, "Inputs must not have scriptSigs and scriptWitnesses", + self.nodes[0].converttopsbt, hexstring=signedtx['hex'], permitsigdata=False) + assert_raises_rpc_error(-22, "Inputs must not have scriptSigs and scriptWitnesses", + self.nodes[0].converttopsbt, hexstring=signedtx['hex'], permitsigdata=False, iswitness=True) # Unless we allow it to convert and strip signatures - self.nodes[0].converttopsbt(signedtx['hex'], True) - - # Explicitly allow converting non-empty txs - new_psbt = self.nodes[0].converttopsbt(rawtx['hex']) - self.nodes[0].decodepsbt(new_psbt) + self.nodes[0].converttopsbt(hexstring=signedtx['hex'], permitsigdata=True) # Create outputs to nodes 1 and 2 node1_addr = self.nodes[1].getnewaddress() @@ -778,7 +779,7 @@ class PSBTTest(BitcoinTestFramework): psbt = wallet.walletcreatefundedpsbt( inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": input_weight}], outputs={self.nodes[0].getnewaddress(): 15}, - options={"add_inputs": True} + add_inputs=True, ) signed = wallet.walletprocesspsbt(psbt["psbt"]) signed = self.nodes[0].walletprocesspsbt(signed["psbt"]) @@ -788,21 +789,21 @@ class PSBTTest(BitcoinTestFramework): psbt2 = wallet.walletcreatefundedpsbt( inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": low_input_weight}], outputs={self.nodes[0].getnewaddress(): 15}, - options={"add_inputs": True} + add_inputs=True, ) assert_greater_than(psbt["fee"], psbt2["fee"]) # Increasing the weight should have a higher fee psbt2 = wallet.walletcreatefundedpsbt( inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], outputs={self.nodes[0].getnewaddress(): 15}, - options={"add_inputs": True} + add_inputs=True, ) assert_greater_than(psbt2["fee"], psbt["fee"]) # The provided weight should override the calculated weight when solving data is provided psbt3 = wallet.walletcreatefundedpsbt( inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], outputs={self.nodes[0].getnewaddress(): 15}, - options={'add_inputs': True, "solving_data":{"descriptors": [desc]}} + add_inputs=True, solving_data={"descriptors": [desc]}, ) assert_equal(psbt2["fee"], psbt3["fee"]) @@ -816,7 +817,7 @@ class PSBTTest(BitcoinTestFramework): psbt3 = wallet.walletcreatefundedpsbt( inputs=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], outputs={self.nodes[0].getnewaddress(): 15}, - options={"add_inputs": True} + add_inputs=True, ) assert_equal(psbt2["fee"], psbt3["fee"]) @@ -944,6 +945,48 @@ class PSBTTest(BitcoinTestFramework): self.log.info("Test we don't crash when making a 0-value funded transaction at 0 fee without forcing an input selection") assert_raises_rpc_error(-4, "Transaction requires one destination of non-0 value, a non-0 feerate, or a pre-selected input", self.nodes[0].walletcreatefundedpsbt, [], [{"data": "deadbeef"}], 0, {"fee_rate": "0"}) + self.log.info("Test descriptorprocesspsbt updates and signs a psbt with descriptors") + + self.generate(self.nodes[2], 1) + + # Disable the wallet for node 2 since `descriptorprocesspsbt` does not use the wallet + self.restart_node(2, extra_args=["-disablewallet"]) + self.connect_nodes(0, 2) + self.connect_nodes(1, 2) + + key_info = get_generate_key() + key = key_info.privkey + address = key_info.p2wpkh_addr + + descriptor = descsum_create(f"wpkh({key})") + + txid = self.nodes[0].sendtoaddress(address, 1) + self.sync_all() + vout = find_output(self.nodes[0], txid, 1) + + psbt = self.nodes[2].createpsbt([{"txid": txid, "vout": vout}], {self.nodes[0].getnewaddress(): 0.99999}) + decoded = self.nodes[2].decodepsbt(psbt) + test_psbt_input_keys(decoded['inputs'][0], []) + + # Test that even if the wrong descriptor is given, `witness_utxo` and `non_witness_utxo` + # are still added to the psbt + alt_descriptor = descsum_create(f"wpkh({get_generate_key().privkey})") + alt_psbt = self.nodes[2].descriptorprocesspsbt(psbt=psbt, descriptors=[alt_descriptor], sighashtype="ALL")["psbt"] + decoded = self.nodes[2].decodepsbt(alt_psbt) + test_psbt_input_keys(decoded['inputs'][0], ['witness_utxo', 'non_witness_utxo']) + + # Test that the psbt is not finalized and does not have bip32_derivs unless specified + psbt = self.nodes[2].descriptorprocesspsbt(psbt=psbt, descriptors=[descriptor], sighashtype="ALL", bip32derivs=True, finalize=False)["psbt"] + decoded = self.nodes[2].decodepsbt(psbt) + test_psbt_input_keys(decoded['inputs'][0], ['witness_utxo', 'non_witness_utxo', 'partial_signatures', 'bip32_derivs']) + + psbt = self.nodes[2].descriptorprocesspsbt(psbt=psbt, descriptors=[descriptor], sighashtype="ALL", bip32derivs=False, finalize=True)["psbt"] + decoded = self.nodes[2].decodepsbt(psbt) + test_psbt_input_keys(decoded['inputs'][0], ['witness_utxo', 'non_witness_utxo', 'final_scriptwitness']) + + # Broadcast transaction + rawtx = self.nodes[2].finalizepsbt(psbt)["hex"] + self.nodes[2].sendrawtransaction(rawtx) if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index f97636aae7..9f77f209ef 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -112,7 +112,7 @@ class ScantxoutsetTest(BitcoinTestFramework): assert_equal(self.nodes[0].scantxoutset("start", [{"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": [1500, 1500]}])['total_amount'], Decimal("16.384")) # Test the reported descriptors for a few matches - assert_equal(descriptors(self.nodes[0].scantxoutset("start", [{"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0'/*)", "range": 1499}])), ["pkh([0c5f9a1e/0'/0'/0]026dbd8b2315f296d36e6b6920b1579ca75569464875c7ebe869b536a7d9503c8c)#dzxw429x", "pkh([0c5f9a1e/0'/0'/1]033e6f25d76c00bedb3a8993c7d5739ee806397f0529b1b31dda31ef890f19a60c)#43rvceed"]) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [{"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0h/*)", "range": 1499}])), ["pkh([0c5f9a1e/0h/0h/0]026dbd8b2315f296d36e6b6920b1579ca75569464875c7ebe869b536a7d9503c8c)#rthll0rg", "pkh([0c5f9a1e/0h/0h/1]033e6f25d76c00bedb3a8993c7d5739ee806397f0529b1b31dda31ef890f19a60c)#mcjajulr"]) assert_equal(descriptors(self.nodes[0].scantxoutset("start", ["combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"])), ["pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8"]) assert_equal(descriptors(self.nodes[0].scantxoutset("start", [{"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])), ['pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8', 'pkh([0c5f9a1e/1/1/1500]03832901c250025da2aebae2bfb38d5c703a57ab66ad477f9c578bfbcd78abca6f)#vchwd07g', 'pkh([0c5f9a1e/1/1/1]030d820fc9e8211c4169be8530efbc632775d8286167afd178caaf1089b77daba7)#z2t3ypsa']) @@ -120,6 +120,9 @@ class ScantxoutsetTest(BitcoinTestFramework): assert_equal(self.nodes[0].scantxoutset("status"), None) assert_equal(self.nodes[0].scantxoutset("abort"), False) + # check that first arg is needed + assert_raises_rpc_error(-1, "scantxoutset \"action\" ( [scanobjects,...] )", self.nodes[0].scantxoutset) + # Check that second arg is needed for start assert_raises_rpc_error(-1, "scanobjects argument is required for the start action", self.nodes[0].scantxoutset, "start") diff --git a/test/functional/rpc_validateaddress.py b/test/functional/rpc_validateaddress.py new file mode 100755 index 0000000000..d87ba098c3 --- /dev/null +++ b/test/functional/rpc_validateaddress.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test validateaddress for main chain""" + +from test_framework.test_framework import BitcoinTestFramework + +from test_framework.util import assert_equal + +INVALID_DATA = [ + # BIP 173 + ( + "tc1qw508d6qejxtdg4y5r3zarvary0c5xw7kg3g4ty", + "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", # Invalid hrp + [], + ), + ("bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t5", "Invalid Bech32 checksum", [41]), + ( + "BC13W508D6QEJXTDG4Y5R3ZARVARY0C5XW7KN40WF2", + "Version 1+ witness address must use Bech32m checksum", + [], + ), + ( + "bc1rw5uspcuh", + "Version 1+ witness address must use Bech32m checksum", # Invalid program length + [], + ), + ( + "bc10w508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7kw5rljs90", + "Version 1+ witness address must use Bech32m checksum", # Invalid program length + [], + ), + ( + "BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P", + "Invalid Bech32 v0 address program size (16 bytes), per BIP141", + [], + ), + ( + "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sL5k7", + "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", # tb1, Mixed case + [], + ), + ( + "BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3t4", + "Invalid character or mixed case", # bc1, Mixed case, not in BIP 173 test vectors + [40], + ), + ( + "bc1zw508d6qejxtdg4y5r3zarvaryvqyzf3du", + "Version 1+ witness address must use Bech32m checksum", # Wrong padding + [], + ), + ( + "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv", + "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", # tb1, Non-zero padding in 8-to-5 conversion + [], + ), + ("bc1gmk9yu", "Empty Bech32 data section", []), + # BIP 350 + ( + "tc1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vq5zuyut", + "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", # Invalid human-readable part + [], + ), + ( + "bc1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vqh2y7hd", + "Version 1+ witness address must use Bech32m checksum", # Invalid checksum (Bech32 instead of Bech32m) + [], + ), + ( + "tb1z0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vqglt7rf", + "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", # tb1, Invalid checksum (Bech32 instead of Bech32m) + [], + ), + ( + "BC1S0XLXVLHEMJA6C4DQV22UAPCTQUPFHLXM9H8Z3K2E72Q4K9HCZ7VQ54WELL", + "Version 1+ witness address must use Bech32m checksum", # Invalid checksum (Bech32 instead of Bech32m) + [], + ), + ( + "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kemeawh", + "Version 0 witness address must use Bech32 checksum", # Invalid checksum (Bech32m instead of Bech32) + [], + ), + ( + "tb1q0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vq24jc47", + "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", # tb1, Invalid checksum (Bech32m instead of Bech32) + [], + ), + ( + "bc1p38j9r5y49hruaue7wxjce0updqjuyyx0kh56v8s25huc6995vvpql3jow4", + "Invalid Base 32 character", # Invalid character in checksum + [59], + ), + ( + "BC130XLXVLHEMJA6C4DQV22UAPCTQUPFHLXM9H8Z3K2E72Q4K9HCZ7VQ7ZWS8R", + "Invalid Bech32 address witness version", + [], + ), + ("bc1pw5dgrnzv", "Invalid Bech32 address program size (1 byte)", []), + ( + "bc1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7v8n0nx0muaewav253zgeav", + "Invalid Bech32 address program size (41 bytes)", + [], + ), + ( + "BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P", + "Invalid Bech32 v0 address program size (16 bytes), per BIP141", + [], + ), + ( + "tb1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vq47Zagq", + "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", # tb1, Mixed case + [], + ), + ( + "bc1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7v07qwwzcrf", + "Invalid padding in Bech32 data section", # zero padding of more than 4 bits + [], + ), + ( + "tb1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vpggkg4j", + "Invalid or unsupported Segwit (Bech32) or Base58 encoding.", # tb1, Non-zero padding in 8-to-5 conversion + [], + ), + ("bc1gmk9yu", "Empty Bech32 data section", []), +] +VALID_DATA = [ + # BIP 350 + ( + "BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4", + "0014751e76e8199196d454941c45d1b3a323f1433bd6", + ), + # ( + # "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7", + # "00201863143c14c5166804bd19203356da136c985678cd4d27a1b8c6329604903262", + # ), + ( + "bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3", + "00201863143c14c5166804bd19203356da136c985678cd4d27a1b8c6329604903262", + ), + ( + "bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7kt5nd6y", + "5128751e76e8199196d454941c45d1b3a323f1433bd6751e76e8199196d454941c45d1b3a323f1433bd6", + ), + ("BC1SW50QGDZ25J", "6002751e"), + ("bc1zw508d6qejxtdg4y5r3zarvaryvaxxpcs", "5210751e76e8199196d454941c45d1b3a323"), + # ( + # "tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy", + # "0020000000c4a5cad46221b2a187905e5266362b99d5e91c6ce24d165dab93e86433", + # ), + ( + "bc1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvses5wp4dt", + "0020000000c4a5cad46221b2a187905e5266362b99d5e91c6ce24d165dab93e86433", + ), + # ( + # "tb1pqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesf3hn0c", + # "5120000000c4a5cad46221b2a187905e5266362b99d5e91c6ce24d165dab93e86433", + # ), + ( + "bc1pqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvses7epu4h", + "5120000000c4a5cad46221b2a187905e5266362b99d5e91c6ce24d165dab93e86433", + ), + ( + "bc1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vqzk5jj0", + "512079be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798", + ), +] + + +class ValidateAddressMainTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.chain = "" # main + self.num_nodes = 1 + self.extra_args = [["-prune=899"]] * self.num_nodes + + def check_valid(self, addr, spk): + info = self.nodes[0].validateaddress(addr) + assert_equal(info["isvalid"], True) + assert_equal(info["scriptPubKey"], spk) + assert "error" not in info + assert "error_locations" not in info + + def check_invalid(self, addr, error_str, error_locations): + res = self.nodes[0].validateaddress(addr) + assert_equal(res["isvalid"], False) + assert_equal(res["error"], error_str) + assert_equal(res["error_locations"], error_locations) + + def test_validateaddress(self): + for (addr, error, locs) in INVALID_DATA: + self.check_invalid(addr, error, locs) + for (addr, spk) in VALID_DATA: + self.check_valid(addr, spk) + + def run_test(self): + self.test_validateaddress() + + +if __name__ == "__main__": + ValidateAddressMainTest().main() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 66a23b443c..d4dc90a517 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -228,6 +228,23 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): PortSeed.n = self.options.port_seed + def set_binary_paths(self): + """Update self.options with the paths of all binaries from environment variables or their default values""" + + binaries = { + "bitcoind": ("bitcoind", "BITCOIND"), + "bitcoin-cli": ("bitcoincli", "BITCOINCLI"), + "bitcoin-util": ("bitcoinutil", "BITCOINUTIL"), + "bitcoin-wallet": ("bitcoinwallet", "BITCOINWALLET"), + } + for binary, [attribute_name, env_variable_name] in binaries.items(): + default_filename = os.path.join( + self.config["environment"]["BUILDDIR"], + "src", + binary + self.config["environment"]["EXEEXT"], + ) + setattr(self.options, attribute_name, os.getenv(env_variable_name, default=default_filename)) + def setup(self): """Call this method to start up the test framework object with options set.""" @@ -237,24 +254,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): config = self.config - fname_bitcoind = os.path.join( - config["environment"]["BUILDDIR"], - "src", - "bitcoind" + config["environment"]["EXEEXT"], - ) - fname_bitcoincli = os.path.join( - config["environment"]["BUILDDIR"], - "src", - "bitcoin-cli" + config["environment"]["EXEEXT"], - ) - fname_bitcoinutil = os.path.join( - config["environment"]["BUILDDIR"], - "src", - "bitcoin-util" + config["environment"]["EXEEXT"], - ) - self.options.bitcoind = os.getenv("BITCOIND", default=fname_bitcoind) - self.options.bitcoincli = os.getenv("BITCOINCLI", default=fname_bitcoincli) - self.options.bitcoinutil = os.getenv("BITCOINUTIL", default=fname_bitcoinutil) + self.set_binary_paths() os.environ['PATH'] = os.pathsep.join([ os.path.join(config['environment']['BUILDDIR'], 'src'), diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 56abe5f26a..4466bd544f 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -190,7 +190,7 @@ class TestNode(): assert self.rpc_connected and self.rpc is not None, self._node_msg("Error: no RPC connection") return getattr(RPCOverloadWrapper(self.rpc, descriptors=self.descriptors), name) - def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, **kwargs): + def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, env=None, **kwargs): """Start the node.""" if extra_args is None: extra_args = self.extra_args @@ -213,6 +213,8 @@ class TestNode(): # add environment variable LIBC_FATAL_STDERR_=1 so that libc errors are written to stderr and not the terminal subp_env = dict(os.environ, LIBC_FATAL_STDERR_="1") + if env is not None: + subp_env.update(env) self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, cwd=cwd, **kwargs) @@ -363,7 +365,7 @@ class TestNode(): if wait_until_stopped: self.wait_until_stopped() - def is_node_stopped(self): + def is_node_stopped(self, expected_ret_code=None): """Checks whether the node has stopped. Returns True if the node has stopped. False otherwise. @@ -375,8 +377,13 @@ class TestNode(): return False # process has stopped. Assert that it didn't return an error code. - assert return_code == 0, self._node_msg( - "Node returned non-zero exit code (%d) when stopping" % return_code) + # unless 'expected_ret_code' is provided. + if expected_ret_code is not None: + assert return_code == expected_ret_code, self._node_msg( + "Node returned unexpected exit code (%d) vs (%d) when stopping" % (return_code, expected_ret_code)) + else: + assert return_code == 0, self._node_msg( + "Node returned non-zero exit code (%d) when stopping" % return_code) self.running = False self.process = None self.rpc_connected = False @@ -384,8 +391,9 @@ class TestNode(): self.log.debug("Node stopped") return True - def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): - wait_until_helper(self.is_node_stopped, timeout=timeout, timeout_factor=self.timeout_factor) + def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT, expect_error=False): + expected_ret_code = 1 if expect_error else None # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS + wait_until_helper(lambda: self.is_node_stopped(expected_ret_code=expected_ret_code), timeout=timeout, timeout_factor=self.timeout_factor) def replace_in_config(self, replacements): """ diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 2c227922c5..d3b3e4d536 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -12,13 +12,15 @@ import inspect import json import logging import os +import pathlib import random import re +import sys import time from . import coverage from .authproxy import AuthServiceProxy, JSONRPCException -from typing import Callable, Optional +from typing import Callable, Optional, Tuple logger = logging.getLogger("TestFramework.utils") @@ -419,6 +421,22 @@ def get_datadir_path(dirname, n): return os.path.join(dirname, "node" + str(n)) +def get_temp_default_datadir(temp_dir: pathlib.Path) -> Tuple[dict, pathlib.Path]: + """Return os-specific environment variables that can be set to make the + GetDefaultDataDir() function return a datadir path under the provided + temp_dir, as well as the complete path it would return.""" + if sys.platform == "win32": + env = dict(APPDATA=str(temp_dir)) + datadir = temp_dir / "Bitcoin" + else: + env = dict(HOME=str(temp_dir)) + if sys.platform == "darwin": + datadir = temp_dir / "Library/Application Support/Bitcoin" + else: + datadir = temp_dir / ".bitcoin" + return env, datadir + + def append_config(datadir, options): with open(os.path.join(datadir, "bitcoin.conf"), 'a', encoding='utf8') as f: for option in options: diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index fcd396c700..1d546e12bd 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -218,10 +218,12 @@ class MiniWallet: txid: get the first utxo we find from a specific transaction """ self._utxos = sorted(self._utxos, key=lambda k: (k['value'], -k['height'])) # Put the largest utxo last + blocks_height = self._test_node.getblockchaininfo()['blocks'] + mature_coins = list(filter(lambda utxo: not utxo['coinbase'] or COINBASE_MATURITY - 1 <= blocks_height - utxo['height'], self._utxos)) if txid: utxo_filter: Any = filter(lambda utxo: txid == utxo['txid'], self._utxos) else: - utxo_filter = reversed(self._utxos) # By default the largest utxo + utxo_filter = reversed(mature_coins) # By default the largest utxo if vout is not None: utxo_filter = filter(lambda utxo: vout == utxo['vout'], utxo_filter) index = self._utxos.index(next(utxo_filter)) @@ -233,7 +235,8 @@ class MiniWallet: def get_utxos(self, *, include_immature_coinbase=False, mark_as_spent=True): """Returns the list of all utxos and optionally mark them as spent""" if not include_immature_coinbase: - utxo_filter = filter(lambda utxo: not utxo['coinbase'] or COINBASE_MATURITY <= utxo['confirmations'], self._utxos) + blocks_height = self._test_node.getblockchaininfo()['blocks'] + utxo_filter = filter(lambda utxo: not utxo['coinbase'] or COINBASE_MATURITY - 1 <= blocks_height - utxo['height'], self._utxos) else: utxo_filter = self._utxos utxos = deepcopy(list(utxo_filter)) @@ -256,15 +259,19 @@ class MiniWallet: Note that this method fails if there is no single internal utxo available that can cover the cost for the amount and the fixed fee (the utxo with the largest value is taken). - - Returns a tuple (txid, n) referring to the created external utxo outpoint. """ tx = self.create_self_transfer(fee_rate=0)["tx"] assert_greater_than_or_equal(tx.vout[0].nValue, amount + fee) tx.vout[0].nValue -= (amount + fee) # change output -> MiniWallet tx.vout.append(CTxOut(amount, scriptPubKey)) # arbitrary output -> to be returned txid = self.sendrawtransaction(from_node=from_node, tx_hex=tx.serialize().hex()) - return txid, 1 + return { + "sent_vout": 1, + "txid": txid, + "wtxid": tx.getwtxid(), + "hex": tx.serialize().hex(), + "tx": tx, + } def send_self_transfer_multi(self, *, from_node, **kwargs): """Call create_self_transfer_multi and send the transaction.""" diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 4fc3d4a4df..c834086b6f 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -173,6 +173,7 @@ BASE_SCRIPTS = [ 'wallet_fast_rescan.py --descriptors', 'interface_zmq.py', 'rpc_invalid_address_message.py', + 'rpc_validateaddress.py', 'interface_bitcoin_cli.py --legacy-wallet', 'interface_bitcoin_cli.py --descriptors', 'feature_bind_extra.py', @@ -197,6 +198,8 @@ BASE_SCRIPTS = [ 'wallet_watchonly.py --legacy-wallet', 'wallet_watchonly.py --usecli --legacy-wallet', 'wallet_reorgsrestore.py', + 'wallet_conflicts.py --legacy-wallet', + 'wallet_conflicts.py --descriptors', 'interface_http.py', 'interface_rpc.py', 'interface_usdt_coinselection.py', @@ -530,6 +533,12 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= # Test Framework Tests print("Running Unit Tests for Test Framework Modules") + + tests_dir = src_dir + '/test/functional/' + # This allows `test_runner.py` to work from an out-of-source build directory using a symlink, + # a hard link or a copy on any platform. See https://github.com/bitcoin/bitcoin/pull/27561. + sys.path.append(tests_dir) + test_framework_tests = unittest.TestSuite() for module in TEST_FRAMEWORK_MODULES: test_framework_tests.addTest(unittest.TestLoader().loadTestsFromName("test_framework.{}".format(module))) @@ -538,8 +547,6 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= logging.debug("Early exiting after failure in TestFramework unit tests") sys.exit(False) - tests_dir = src_dir + '/test/functional/' - flags = ['--cachedir={}'.format(cache_dir)] + args if enable_coverage: diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index a888f93b03..95999649b4 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -32,12 +32,11 @@ class ToolWalletTest(BitcoinTestFramework): self.skip_if_no_wallet_tool() def bitcoin_wallet_process(self, *args): - binary = self.config["environment"]["BUILDDIR"] + '/src/bitcoin-wallet' + self.config["environment"]["EXEEXT"] default_args = ['-datadir={}'.format(self.nodes[0].datadir), '-chain=%s' % self.chain] if not self.options.descriptors and 'create' in args: default_args.append('-legacy') - return subprocess.Popen([binary] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + return subprocess.Popen([self.options.bitcoinwallet] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) def assert_raises_tool_error(self, error, *args): p = self.bitcoin_wallet_process(*args) diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 934f44588d..2691507773 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -226,20 +226,16 @@ class AbandonConflictTest(BitcoinTestFramework): assert_equal(double_spend["walletconflicts"], [txAB1]) # Verify that B and C's 10 BTC outputs are available for spending again because AB1 is now conflicted + assert_equal(alice.gettransaction(txAB1)["confirmations"], -1) newbalance = alice.getbalance() assert_equal(newbalance, balance + Decimal("20")) balance = newbalance - # There is currently a minor bug around this and so this test doesn't work. See Issue #7315 - # Invalidate the block with the double spend and B's 10 BTC output should no longer be available - # Don't think C's should either + # Invalidate the block with the double spend. B & C's 10 BTC outputs should no longer be available self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) + assert_equal(alice.gettransaction(txAB1)["confirmations"], 0) newbalance = alice.getbalance() - #assert_equal(newbalance, balance - Decimal("10")) - self.log.info("If balance has not declined after invalidateblock then out of mempool wallet tx which is no longer") - self.log.info("conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315") - assert_equal(balance, newbalance) - + assert_equal(newbalance, balance - Decimal("20")) if __name__ == '__main__': AbandonConflictTest().main() diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index ebeb5620e5..be5b3ebadb 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -176,7 +176,7 @@ class AddressTypeTest(BitcoinTestFramework): for deriv in decode['inputs'][0]['bip32_derivs']: assert_equal(len(deriv['master_fingerprint']), 8) assert_equal(deriv['path'][0], 'm') - key_descs[deriv['pubkey']] = '[' + deriv['master_fingerprint'] + deriv['path'][1:] + ']' + deriv['pubkey'] + key_descs[deriv['pubkey']] = '[' + deriv['master_fingerprint'] + deriv['path'][1:].replace("'","h") + ']' + deriv['pubkey'] # Verify the descriptor checksum against the Python implementation assert descsum_check(info['desc']) diff --git a/test/functional/wallet_backwards_compatibility.py b/test/functional/wallet_backwards_compatibility.py index a6401a76c1..5088e11eda 100755 --- a/test/functional/wallet_backwards_compatibility.py +++ b/test/functional/wallet_backwards_compatibility.py @@ -253,7 +253,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): wallet = node_v17.get_wallet_rpc("u1_v17") address = wallet.getnewaddress("bech32") v17_info = wallet.getaddressinfo(address) - hdkeypath = v17_info["hdkeypath"] + hdkeypath = v17_info["hdkeypath"].replace("'", "h") pubkey = v17_info["pubkey"] if self.is_bdb_compiled(): diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index e23ec7bc01..a1b805c09e 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -8,6 +8,10 @@ from itertools import product from test_framework.blocktools import COINBASE_MATURITY from test_framework.descriptors import descsum_create +from test_framework.messages import ( + COIN, + DEFAULT_ANCESTOR_LIMIT, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_array_result, @@ -17,6 +21,7 @@ from test_framework.util import ( find_vout_for_address, ) from test_framework.wallet_util import test_address +from test_framework.wallet import MiniWallet NOT_A_NUMBER_OR_STRING = "Amount is not a number or string" OUT_OF_RANGE = "Amount out of range" @@ -784,6 +789,34 @@ class WalletTest(BitcoinTestFramework): zeroconf_wallet.sendtoaddress(zeroconf_wallet.getnewaddress(), Decimal('0.5')) + self.test_chain_listunspent() + + def test_chain_listunspent(self): + if not self.options.descriptors: + return + self.wallet = MiniWallet(self.nodes[0]) + self.nodes[0].get_wallet_rpc(self.default_wallet_name).sendtoaddress(self.wallet.get_address(), "5") + self.generate(self.wallet, 1, sync_fun=self.no_op) + self.nodes[0].createwallet("watch_wallet", disable_private_keys=True) + watch_wallet = self.nodes[0].get_wallet_rpc("watch_wallet") + watch_wallet.importaddress(self.wallet.get_address()) + + # DEFAULT_ANCESTOR_LIMIT transactions off a confirmed tx should be fine + chain = self.wallet.create_self_transfer_chain(chain_length=DEFAULT_ANCESTOR_LIMIT) + ancestor_vsize = 0 + ancestor_fees = Decimal(0) + + for i, t in enumerate(chain): + ancestor_vsize += t["tx"].get_vsize() + ancestor_fees += t["fee"] + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=t["hex"]) + # Check that listunspent ancestor{count, size, fees} yield the correct results + wallet_unspent = watch_wallet.listunspent(minconf=0) + this_unspent = next(utxo_info for utxo_info in wallet_unspent if utxo_info["txid"] == t["txid"]) + assert_equal(this_unspent['ancestorcount'], i + 1) + assert_equal(this_unspent['ancestorsize'], ancestor_vsize) + assert_equal(this_unspent['ancestorfees'], ancestor_fees * COIN) + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 19c8022600..b9ebf64c22 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -123,36 +123,36 @@ class BumpFeeTest(BitcoinTestFramework): assert_raises_rpc_error(-3, "Unexpected key {}".format(key), rbf_node.bumpfee, rbfid, {key: NORMAL}) # Bumping to just above minrelay should fail to increase the total fee enough. - assert_raises_rpc_error(-8, "Insufficient total fee 0.00000141", rbf_node.bumpfee, rbfid, {"fee_rate": INSUFFICIENT}) + assert_raises_rpc_error(-8, "Insufficient total fee 0.00000141", rbf_node.bumpfee, rbfid, fee_rate=INSUFFICIENT) self.log.info("Test invalid fee rate settings") assert_raises_rpc_error(-4, "Specified or calculated fee 0.141 is too high (cannot be higher than -maxtxfee 0.10", - rbf_node.bumpfee, rbfid, {"fee_rate": TOO_HIGH}) + rbf_node.bumpfee, rbfid, fee_rate=TOO_HIGH) # Test fee_rate with zero values. msg = "Insufficient total fee 0.00" for zero_value in [0, 0.000, 0.00000000, "0", "0.000", "0.00000000"]: - assert_raises_rpc_error(-8, msg, rbf_node.bumpfee, rbfid, {"fee_rate": zero_value}) + assert_raises_rpc_error(-8, msg, rbf_node.bumpfee, rbfid, fee_rate=zero_value) msg = "Invalid amount" # Test fee_rate values that don't pass fixed-point parsing checks. for invalid_value in ["", 0.000000001, 1e-09, 1.111111111, 1111111111111111, "31.999999999999999999999"]: - assert_raises_rpc_error(-3, msg, rbf_node.bumpfee, rbfid, {"fee_rate": invalid_value}) + assert_raises_rpc_error(-3, msg, rbf_node.bumpfee, rbfid, fee_rate=invalid_value) # Test fee_rate values that cannot be represented in sat/vB. for invalid_value in [0.0001, 0.00000001, 0.00099999, 31.99999999, "0.0001", "0.00000001", "0.00099999", "31.99999999"]: - assert_raises_rpc_error(-3, msg, rbf_node.bumpfee, rbfid, {"fee_rate": invalid_value}) + assert_raises_rpc_error(-3, msg, rbf_node.bumpfee, rbfid, fee_rate=invalid_value) # Test fee_rate out of range (negative number). - assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate": -1}) + assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, fee_rate=-1) # Test type error. for value in [{"foo": "bar"}, True]: - assert_raises_rpc_error(-3, "Amount is not a number or string", rbf_node.bumpfee, rbfid, {"fee_rate": value}) + assert_raises_rpc_error(-3, "Amount is not a number or string", rbf_node.bumpfee, rbfid, fee_rate=value) self.log.info("Test explicit fee rate raises RPC error if both fee_rate and conf_target are passed") assert_raises_rpc_error(-8, "Cannot specify both conf_target and fee_rate. Please provide either a confirmation " "target in blocks for automatic fee estimation, or an explicit fee rate.", - rbf_node.bumpfee, rbfid, {"conf_target": NORMAL, "fee_rate": NORMAL}) + rbf_node.bumpfee, rbfid, conf_target=NORMAL, fee_rate=NORMAL) self.log.info("Test explicit fee rate raises RPC error if both fee_rate and estimate_mode are passed") assert_raises_rpc_error(-8, "Cannot specify both estimate_mode and fee_rate", - rbf_node.bumpfee, rbfid, {"estimate_mode": "economical", "fee_rate": NORMAL}) + rbf_node.bumpfee, rbfid, estimate_mode="economical", fee_rate=NORMAL) self.log.info("Test invalid conf_target settings") assert_raises_rpc_error(-8, "confTarget and conf_target options should not both be set", @@ -161,10 +161,10 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Test invalid estimate_mode settings") for k, v in {"number": 42, "object": {"foo": "bar"}}.items(): assert_raises_rpc_error(-3, f"JSON value of type {k} for field estimate_mode is not of expected type string", - rbf_node.bumpfee, rbfid, {"estimate_mode": v}) + rbf_node.bumpfee, rbfid, estimate_mode=v) for mode in ["foo", Decimal("3.1415"), "sat/B", "BTC/kB"]: assert_raises_rpc_error(-8, 'Invalid estimate_mode parameter, must be one of: "unset", "economical", "conservative"', - rbf_node.bumpfee, rbfid, {"estimate_mode": mode}) + rbf_node.bumpfee, rbfid, estimate_mode=mode) self.log.info("Test invalid outputs values") assert_raises_rpc_error(-8, "Invalid parameter, output argument cannot be an empty array", @@ -232,12 +232,12 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): self.sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() if mode == "fee_rate": - bumped_psbt = rbf_node.psbtbumpfee(rbfid, {"fee_rate": str(NORMAL)}) - bumped_tx = rbf_node.bumpfee(rbfid, {"fee_rate": NORMAL}) + bumped_psbt = rbf_node.psbtbumpfee(rbfid, fee_rate=str(NORMAL)) + bumped_tx = rbf_node.bumpfee(rbfid, fee_rate=NORMAL) elif mode == "new_outputs": new_address = peer_node.getnewaddress() - bumped_psbt = rbf_node.psbtbumpfee(rbfid, {"outputs": {new_address: 0.0003}}) - bumped_tx = rbf_node.bumpfee(rbfid, {"outputs": {new_address: 0.0003}}) + bumped_psbt = rbf_node.psbtbumpfee(rbfid, outputs={new_address: 0.0003}) + bumped_tx = rbf_node.bumpfee(rbfid, outputs={new_address: 0.0003}) else: bumped_psbt = rbf_node.psbtbumpfee(rbfid) bumped_tx = rbf_node.bumpfee(rbfid) @@ -305,7 +305,7 @@ def test_notmine_bumpfee(self, rbf_node, peer_node, dest_address): # Note that this test depends upon the RPC code checking input ownership prior to change outputs # (since it can't use fundrawtransaction, it lacks a proper change output) fee = Decimal("0.001") - utxos = [node.listunspent(query_options={'minimumAmount': fee})[-1] for node in (rbf_node, peer_node)] + utxos = [node.listunspent(minimumAmount=fee)[-1] for node in (rbf_node, peer_node)] inputs = [{ "txid": utxo["txid"], "vout": utxo["vout"], @@ -335,7 +335,7 @@ def test_notmine_bumpfee(self, rbf_node, peer_node, dest_address): psbt = rbf_node.psbtbumpfee(txid=rbfid) finish_psbtbumpfee(psbt["psbt"]) - psbt = rbf_node.psbtbumpfee(txid=rbfid, options={"fee_rate": old_feerate + 10}) + psbt = rbf_node.psbtbumpfee(txid=rbfid, fee_rate=old_feerate + 10) finish_psbtbumpfee(psbt["psbt"]) self.clear_mempool() @@ -445,7 +445,7 @@ def test_dust_to_fee(self, rbf_node, dest_address): # Expected fee is 141 vbytes * fee_rate 0.00350250 BTC / 1000 vbytes = 0.00049385 BTC. # or occasionally 140 vbytes * fee_rate 0.00350250 BTC / 1000 vbytes = 0.00049035 BTC. # Dust should be dropped to the fee, so actual bump fee is 0.00050000 BTC. - bumped_tx = rbf_node.bumpfee(rbfid, {"fee_rate": 350.25}) + bumped_tx = rbf_node.bumpfee(rbfid, fee_rate=350.25) full_bumped_tx = rbf_node.getrawtransaction(bumped_tx["txid"], 1) assert_equal(bumped_tx["fee"], Decimal("0.00050000")) assert_equal(len(fulltx["vout"]), 2) @@ -569,7 +569,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): assert_raises_rpc_error(-4, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead.", watcher.bumpfee, original_txid) # Bump fee, obnoxiously high to add additional watchonly input - bumped_psbt = watcher.psbtbumpfee(original_txid, {"fee_rate": HIGH}) + bumped_psbt = watcher.psbtbumpfee(original_txid, fee_rate=HIGH) assert_greater_than(len(watcher.decodepsbt(bumped_psbt['psbt'])["tx"]["vin"]), 1) assert "txid" not in bumped_psbt assert_equal(bumped_psbt["origfee"], -watcher.gettransaction(original_txid)["fee"]) @@ -593,17 +593,17 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): def test_rebumping(self, rbf_node, dest_address): self.log.info('Test that re-bumping the original tx fails, but bumping successor works') rbfid = spend_one_input(rbf_node, dest_address) - bumped = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL}) + bumped = rbf_node.bumpfee(rbfid, fee_rate=ECONOMICAL) assert_raises_rpc_error(-4, f"Cannot bump transaction {rbfid} which was already bumped by transaction {bumped['txid']}", - rbf_node.bumpfee, rbfid, {"fee_rate": NORMAL}) - rbf_node.bumpfee(bumped["txid"], {"fee_rate": NORMAL}) + rbf_node.bumpfee, rbfid, fee_rate=NORMAL) + rbf_node.bumpfee(bumped["txid"], fee_rate=NORMAL) self.clear_mempool() def test_rebumping_not_replaceable(self, rbf_node, dest_address): self.log.info('Test that re-bumping non-replaceable fails') rbfid = spend_one_input(rbf_node, dest_address) - bumped = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL, "replaceable": False}) + bumped = rbf_node.bumpfee(rbfid, fee_rate=ECONOMICAL, replaceable=False) assert_raises_rpc_error(-4, "Transaction is not BIP 125 replaceable", rbf_node.bumpfee, bumped["txid"], {"fee_rate": NORMAL}) self.clear_mempool() @@ -615,7 +615,7 @@ def test_bumpfee_already_spent(self, rbf_node, dest_address): self.generate(rbf_node, 1) # spend coin simply by mining block with tx spent_input = rbf_node.gettransaction(txid=txid, verbose=True)['decoded']['vin'][0] assert_raises_rpc_error(-1, f"{spent_input['txid']}:{spent_input['vout']} is already spent", - rbf_node.bumpfee, txid, {"fee_rate": NORMAL}) + rbf_node.bumpfee, txid, fee_rate=NORMAL) def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): @@ -694,7 +694,7 @@ def test_change_script_match(self, rbf_node, dest_address): assert_equal(len(change_addresses), 1) # Now find that address in each subsequent tx, and no other change - bumped_total_tx = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL}) + bumped_total_tx = rbf_node.bumpfee(rbfid, fee_rate=ECONOMICAL) assert_equal(change_addresses, get_change_address(bumped_total_tx['txid'], rbf_node)) bumped_rate_tx = rbf_node.bumpfee(bumped_total_tx["txid"]) assert_equal(change_addresses, get_change_address(bumped_rate_tx['txid'], rbf_node)) diff --git a/test/functional/wallet_conflicts.py b/test/functional/wallet_conflicts.py new file mode 100755 index 0000000000..802b718cd5 --- /dev/null +++ b/test/functional/wallet_conflicts.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Test that wallet correctly tracks transactions that have been conflicted by blocks, particularly during reorgs. +""" + +from decimal import Decimal + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + +class TxConflicts(BitcoinTestFramework): + def add_options(self, parser): + self.add_wallet_options(parser) + + def set_test_params(self): + self.num_nodes = 3 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def get_utxo_of_value(self, from_tx_id, search_value): + return next(tx_out["vout"] for tx_out in self.nodes[0].gettransaction(from_tx_id)["details"] if tx_out["amount"] == Decimal(f"{search_value}")) + + def run_test(self): + self.log.info("Send tx from which to conflict outputs later") + txid_conflict_from_1 = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) + txid_conflict_from_2 = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) + self.generate(self.nodes[0], 1) + self.sync_blocks() + + self.log.info("Disconnect nodes to broadcast conflicts on their respective chains") + self.disconnect_nodes(0, 1) + self.disconnect_nodes(2, 1) + + self.log.info("Create transactions that conflict with each other") + output_A = self.get_utxo_of_value(from_tx_id=txid_conflict_from_1, search_value=10) + output_B = self.get_utxo_of_value(from_tx_id=txid_conflict_from_2, search_value=10) + + # First create a transaction that consumes both A and B outputs. + # + # | tx1 | -----> | | | | + # | AB_parent_tx | ----> | Child_Tx | + # | tx2 | -----> | | | | + # + inputs_tx_AB_parent = [{"txid": txid_conflict_from_1, "vout": output_A}, {"txid": txid_conflict_from_2, "vout": output_B}] + tx_AB_parent = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs_tx_AB_parent, {self.nodes[0].getnewaddress(): Decimal("19.99998")})) + + # Secondly, create two transactions: One consuming output_A, and another one consuming output_B + # + # | tx1 | -----> | Tx_A_1 | + # ---------------- + # | tx2 | -----> | Tx_B_1 | + # + inputs_tx_A_1 = [{"txid": txid_conflict_from_1, "vout": output_A}] + inputs_tx_B_1 = [{"txid": txid_conflict_from_2, "vout": output_B}] + tx_A_1 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs_tx_A_1, {self.nodes[0].getnewaddress(): Decimal("9.99998")})) + tx_B_1 = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs_tx_B_1, {self.nodes[0].getnewaddress(): Decimal("9.99998")})) + + self.log.info("Broadcast conflicted transaction") + txid_AB_parent = self.nodes[0].sendrawtransaction(tx_AB_parent["hex"]) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) + + # Now that 'AB_parent_tx' was broadcast, build 'Child_Tx' + output_c = self.get_utxo_of_value(from_tx_id=txid_AB_parent, search_value=19.99998) + inputs_tx_C_child = [({"txid": txid_AB_parent, "vout": output_c})] + + tx_C_child = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs_tx_C_child, {self.nodes[0].getnewaddress() : Decimal("19.99996")})) + tx_C_child_txid = self.nodes[0].sendrawtransaction(tx_C_child["hex"]) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) + + self.log.info("Broadcast conflicting tx to node 1 and generate a longer chain") + conflicting_txid_A = self.nodes[1].sendrawtransaction(tx_A_1["hex"]) + self.generate(self.nodes[1], 4, sync_fun=self.no_op) + conflicting_txid_B = self.nodes[1].sendrawtransaction(tx_B_1["hex"]) + self.generate(self.nodes[1], 4, sync_fun=self.no_op) + + self.log.info("Connect nodes 0 and 1, trigger reorg and ensure that the tx is effectively conflicted") + self.connect_nodes(0, 1) + self.sync_blocks([self.nodes[0], self.nodes[1]]) + conflicted_AB_tx = self.nodes[0].gettransaction(txid_AB_parent) + tx_C_child = self.nodes[0].gettransaction(tx_C_child_txid) + conflicted_A_tx = self.nodes[0].gettransaction(conflicting_txid_A) + + self.log.info("Verify, after the reorg, that Tx_A was accepted, and tx_AB and its Child_Tx are conflicting now") + # Tx A was accepted, Tx AB was not. + assert conflicted_AB_tx["confirmations"] < 0 + assert conflicted_A_tx["confirmations"] > 0 + + # Conflicted tx should have confirmations set to the confirmations of the most conflicting tx + assert_equal(-conflicted_AB_tx["confirmations"], conflicted_A_tx["confirmations"]) + # Child should inherit conflicted state from parent + assert_equal(-tx_C_child["confirmations"], conflicted_A_tx["confirmations"]) + # Check the confirmations of the conflicting transactions + assert_equal(conflicted_A_tx["confirmations"], 8) + assert_equal(self.nodes[0].gettransaction(conflicting_txid_B)["confirmations"], 4) + + self.log.info("Now generate a longer chain that does not contain any tx") + # Node2 chain without conflicts + self.generate(self.nodes[2], 15, sync_fun=self.no_op) + + # Connect node0 and node2 and wait reorg + self.connect_nodes(0, 2) + self.sync_blocks() + conflicted = self.nodes[0].gettransaction(txid_AB_parent) + tx_C_child = self.nodes[0].gettransaction(tx_C_child_txid) + + self.log.info("Test that formerly conflicted transaction are inactive after reorg") + # Former conflicted tx should be unconfirmed as it hasn't been yet rebroadcast + assert_equal(conflicted["confirmations"], 0) + # Former conflicted child tx should be unconfirmed as it hasn't been rebroadcast + assert_equal(tx_C_child["confirmations"], 0) + # Rebroadcast former conflicted tx and check it confirms smoothly + self.nodes[2].sendrawtransaction(conflicted["hex"]) + self.generate(self.nodes[2], 1) + self.sync_blocks() + former_conflicted = self.nodes[0].gettransaction(txid_AB_parent) + assert_equal(former_conflicted["confirmations"], 1) + assert_equal(former_conflicted["blockheight"], 217) + +if __name__ == '__main__': + TxConflicts().main() diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index b0e93df36a..4673eb091c 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -60,43 +60,43 @@ class WalletDescriptorTest(BitcoinTestFramework): addr = self.nodes[0].getnewaddress("", "legacy") addr_info = self.nodes[0].getaddressinfo(addr) assert addr_info['desc'].startswith('pkh(') - assert_equal(addr_info['hdkeypath'], 'm/44\'/1\'/0\'/0/0') + assert_equal(addr_info['hdkeypath'], 'm/44h/1h/0h/0/0') addr = self.nodes[0].getnewaddress("", "p2sh-segwit") addr_info = self.nodes[0].getaddressinfo(addr) assert addr_info['desc'].startswith('sh(wpkh(') - assert_equal(addr_info['hdkeypath'], 'm/49\'/1\'/0\'/0/0') + assert_equal(addr_info['hdkeypath'], 'm/49h/1h/0h/0/0') addr = self.nodes[0].getnewaddress("", "bech32") addr_info = self.nodes[0].getaddressinfo(addr) assert addr_info['desc'].startswith('wpkh(') - assert_equal(addr_info['hdkeypath'], 'm/84\'/1\'/0\'/0/0') + assert_equal(addr_info['hdkeypath'], 'm/84h/1h/0h/0/0') addr = self.nodes[0].getnewaddress("", "bech32m") addr_info = self.nodes[0].getaddressinfo(addr) assert addr_info['desc'].startswith('tr(') - assert_equal(addr_info['hdkeypath'], 'm/86\'/1\'/0\'/0/0') + assert_equal(addr_info['hdkeypath'], 'm/86h/1h/0h/0/0') # Check that getrawchangeaddress works addr = self.nodes[0].getrawchangeaddress("legacy") addr_info = self.nodes[0].getaddressinfo(addr) assert addr_info['desc'].startswith('pkh(') - assert_equal(addr_info['hdkeypath'], 'm/44\'/1\'/0\'/1/0') + assert_equal(addr_info['hdkeypath'], 'm/44h/1h/0h/1/0') addr = self.nodes[0].getrawchangeaddress("p2sh-segwit") addr_info = self.nodes[0].getaddressinfo(addr) assert addr_info['desc'].startswith('sh(wpkh(') - assert_equal(addr_info['hdkeypath'], 'm/49\'/1\'/0\'/1/0') + assert_equal(addr_info['hdkeypath'], 'm/49h/1h/0h/1/0') addr = self.nodes[0].getrawchangeaddress("bech32") addr_info = self.nodes[0].getaddressinfo(addr) assert addr_info['desc'].startswith('wpkh(') - assert_equal(addr_info['hdkeypath'], 'm/84\'/1\'/0\'/1/0') + assert_equal(addr_info['hdkeypath'], 'm/84h/1h/0h/1/0') addr = self.nodes[0].getrawchangeaddress("bech32m") addr_info = self.nodes[0].getaddressinfo(addr) assert addr_info['desc'].startswith('tr(') - assert_equal(addr_info['hdkeypath'], 'm/86\'/1\'/0\'/1/0') + assert_equal(addr_info['hdkeypath'], 'm/86h/1h/0h/1/0') # Make a wallet to receive coins at self.nodes[0].createwallet(wallet_name="desc2", descriptors=True) @@ -178,14 +178,14 @@ class WalletDescriptorTest(BitcoinTestFramework): self.nodes[0].createwallet(wallet_name='desc_import', disable_private_keys=True, descriptors=True) imp_rpc = self.nodes[0].get_wallet_rpc('desc_import') - addr_types = [('legacy', False, 'pkh(', '44\'/1\'/0\'', -13), - ('p2sh-segwit', False, 'sh(wpkh(', '49\'/1\'/0\'', -14), - ('bech32', False, 'wpkh(', '84\'/1\'/0\'', -13), - ('bech32m', False, 'tr(', '86\'/1\'/0\'', -13), - ('legacy', True, 'pkh(', '44\'/1\'/0\'', -13), - ('p2sh-segwit', True, 'sh(wpkh(', '49\'/1\'/0\'', -14), - ('bech32', True, 'wpkh(', '84\'/1\'/0\'', -13), - ('bech32m', True, 'tr(', '86\'/1\'/0\'', -13)] + addr_types = [('legacy', False, 'pkh(', '44h/1h/0h', -13), + ('p2sh-segwit', False, 'sh(wpkh(', '49h/1h/0h', -14), + ('bech32', False, 'wpkh(', '84h/1h/0h', -13), + ('bech32m', False, 'tr(', '86h/1h/0h', -13), + ('legacy', True, 'pkh(', '44h/1h/0h', -13), + ('p2sh-segwit', True, 'sh(wpkh(', '49h/1h/0h', -14), + ('bech32', True, 'wpkh(', '84h/1h/0h', -13), + ('bech32m', True, 'tr(', '86h/1h/0h', -13)] for addr_type, internal, desc_prefix, deriv_path, int_idx in addr_types: int_str = 'internal' if internal else 'external' diff --git a/test/functional/wallet_fundrawtransaction.py b/test/functional/wallet_fundrawtransaction.py index 29ddb77b41..c88e0b3f6e 100755 --- a/test/functional/wallet_fundrawtransaction.py +++ b/test/functional/wallet_fundrawtransaction.py @@ -154,7 +154,7 @@ class RawTransactionsTest(BitcoinTestFramework): """Ensure setting changePosition in fundraw with an exact match is handled properly.""" self.log.info("Test fundrawtxn changePosition option") rawmatch = self.nodes[2].createrawtransaction([], {self.nodes[2].getnewaddress():50}) - rawmatch = self.nodes[2].fundrawtransaction(rawmatch, {"changePosition":1, "subtractFeeFromOutputs":[0]}) + rawmatch = self.nodes[2].fundrawtransaction(rawmatch, changePosition=1, subtractFeeFromOutputs=[0]) assert_equal(rawmatch["changepos"], -1) self.nodes[3].createwallet(wallet_name="wwatch", disable_private_keys=True) @@ -268,10 +268,10 @@ class RawTransactionsTest(BitcoinTestFramework): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - assert_raises_rpc_error(-3, "Unexpected key foo", self.nodes[2].fundrawtransaction, rawtx, {'foo':'bar'}) + assert_raises_rpc_error(-8, "Unknown named parameter foo", self.nodes[2].fundrawtransaction, rawtx, foo='bar') # reserveChangeKey was deprecated and is now removed - assert_raises_rpc_error(-3, "Unexpected key reserveChangeKey", lambda: self.nodes[2].fundrawtransaction(hexstring=rawtx, options={'reserveChangeKey': True})) + assert_raises_rpc_error(-8, "Unknown named parameter reserveChangeKey", lambda: self.nodes[2].fundrawtransaction(hexstring=rawtx, reserveChangeKey=True)) def test_invalid_change_address(self): self.log.info("Test fundrawtxn with an invalid change address") @@ -283,7 +283,7 @@ class RawTransactionsTest(BitcoinTestFramework): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - assert_raises_rpc_error(-5, "Change address must be a valid bitcoin address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) + assert_raises_rpc_error(-5, "Change address must be a valid bitcoin address", self.nodes[2].fundrawtransaction, rawtx, changeAddress='foobar') def test_valid_change_address(self): self.log.info("Test fundrawtxn with a provided change address") @@ -296,8 +296,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) change = self.nodes[2].getnewaddress() - assert_raises_rpc_error(-8, "changePosition out of bounds", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':change, 'changePosition':2}) - rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {'changeAddress': change, 'changePosition': 0}) + assert_raises_rpc_error(-8, "changePosition out of bounds", self.nodes[2].fundrawtransaction, rawtx, changeAddress=change, changePosition=2) + rawtxfund = self.nodes[2].fundrawtransaction(rawtx, changeAddress=change, changePosition=0) dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) out = dec_tx['vout'][0] assert_equal(change, out['scriptPubKey']['address']) @@ -309,9 +309,9 @@ class RawTransactionsTest(BitcoinTestFramework): inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] outputs = { self.nodes[0].getnewaddress() : Decimal(4.0) } rawtx = self.nodes[2].createrawtransaction(inputs, outputs) - assert_raises_rpc_error(-3, "JSON value of type null is not of expected type string", self.nodes[2].fundrawtransaction, rawtx, {'change_type': None}) - assert_raises_rpc_error(-5, "Unknown change type ''", self.nodes[2].fundrawtransaction, rawtx, {'change_type': ''}) - rawtx = self.nodes[2].fundrawtransaction(rawtx, {'change_type': 'bech32'}) + assert_raises_rpc_error(-3, "JSON value of type null is not of expected type string", self.nodes[2].fundrawtransaction, rawtx, change_type=None) + assert_raises_rpc_error(-5, "Unknown change type ''", self.nodes[2].fundrawtransaction, rawtx, change_type='') + rawtx = self.nodes[2].fundrawtransaction(rawtx, change_type='bech32') dec_tx = self.nodes[2].decoderawtransaction(rawtx['hex']) assert_equal('witness_v0_keyhash', dec_tx['vout'][rawtx['changepos']]['scriptPubKey']['type']) @@ -331,7 +331,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal("00", dec_tx['vin'][0]['scriptSig']['hex']) # Should fail without add_inputs: - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, {"add_inputs": False}) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, add_inputs=False) # add_inputs is enabled by default rawtxfund = self.nodes[2].fundrawtransaction(rawtx) @@ -363,8 +363,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) # Should fail without add_inputs: - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, {"add_inputs": False}) - rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {"add_inputs": True}) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, add_inputs=False) + rawtxfund = self.nodes[2].fundrawtransaction(rawtx, add_inputs=True) dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) totalOut = 0 @@ -397,8 +397,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) # Should fail without add_inputs: - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, {"add_inputs": False}) - rawtxfund = self.nodes[2].fundrawtransaction(rawtx, {"add_inputs": True}) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, self.nodes[2].fundrawtransaction, rawtx, add_inputs=False) + rawtxfund = self.nodes[2].fundrawtransaction(rawtx, add_inputs=True) dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) totalOut = 0 @@ -567,7 +567,7 @@ class RawTransactionsTest(BitcoinTestFramework): oldBalance = self.nodes[1].getbalance() inputs = [] outputs = {self.nodes[1].getnewaddress():1.1} - funded_psbt = wmulti.walletcreatefundedpsbt(inputs=inputs, outputs=outputs, options={'changeAddress': w2.getrawchangeaddress()})['psbt'] + funded_psbt = wmulti.walletcreatefundedpsbt(inputs=inputs, outputs=outputs, changeAddress=w2.getrawchangeaddress())['psbt'] signed_psbt = w2.walletprocesspsbt(funded_psbt) final_psbt = w2.finalizepsbt(signed_psbt['psbt']) @@ -750,7 +750,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[3].loadwallet('wwatch') wwatch = self.nodes[3].get_wallet_rpc('wwatch') w3 = self.nodes[3].get_wallet_rpc(self.default_wallet_name) - result = wwatch.fundrawtransaction(rawtx, {'includeWatching': True, 'changeAddress': w3.getrawchangeaddress(), 'subtractFeeFromOutputs': [0]}) + result = wwatch.fundrawtransaction(rawtx, includeWatching=True, changeAddress=w3.getrawchangeaddress(), subtractFeeFromOutputs=[0]) res_dec = self.nodes[0].decoderawtransaction(result["hex"]) assert_equal(len(res_dec["vin"]), 1) assert res_dec["vin"][0]["txid"] == self.watchonly_txid @@ -779,10 +779,10 @@ class RawTransactionsTest(BitcoinTestFramework): result = node.fundrawtransaction(rawtx) # uses self.min_relay_tx_fee (set by settxfee) btc_kvb_to_sat_vb = 100000 # (1e5) - result1 = node.fundrawtransaction(rawtx, {"fee_rate": str(2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee)}) - result2 = node.fundrawtransaction(rawtx, {"feeRate": 2 * self.min_relay_tx_fee}) - result3 = node.fundrawtransaction(rawtx, {"fee_rate": 10 * btc_kvb_to_sat_vb * self.min_relay_tx_fee}) - result4 = node.fundrawtransaction(rawtx, {"feeRate": str(10 * self.min_relay_tx_fee)}) + result1 = node.fundrawtransaction(rawtx, fee_rate=str(2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee)) + result2 = node.fundrawtransaction(rawtx, feeRate=2 * self.min_relay_tx_fee) + result3 = node.fundrawtransaction(rawtx, fee_rate=10 * btc_kvb_to_sat_vb * self.min_relay_tx_fee) + result4 = node.fundrawtransaction(rawtx, feeRate=str(10 * self.min_relay_tx_fee)) result_fee_rate = result['fee'] * 1000 / count_bytes(result['hex']) assert_fee_amount(result1['fee'], count_bytes(result1['hex']), 2 * result_fee_rate) @@ -797,54 +797,54 @@ class RawTransactionsTest(BitcoinTestFramework): # With no arguments passed, expect fee of 141 satoshis. assert_approx(node.fundrawtransaction(rawtx)["fee"], vexp=0.00000141, vspan=0.00000001) # Expect fee to be 10,000x higher when an explicit fee rate 10,000x greater is specified. - result = node.fundrawtransaction(rawtx, {"fee_rate": 10000}) + result = node.fundrawtransaction(rawtx, fee_rate=10000) assert_approx(result["fee"], vexp=0.0141, vspan=0.0001) self.log.info("Test fundrawtxn with invalid estimate_mode settings") for k, v in {"number": 42, "object": {"foo": "bar"}}.items(): assert_raises_rpc_error(-3, f"JSON value of type {k} for field estimate_mode is not of expected type string", - node.fundrawtransaction, rawtx, {"estimate_mode": v, "conf_target": 0.1, "add_inputs": True}) + node.fundrawtransaction, rawtx, estimate_mode=v, conf_target=0.1, add_inputs=True) for mode in ["", "foo", Decimal("3.141592")]: assert_raises_rpc_error(-8, 'Invalid estimate_mode parameter, must be one of: "unset", "economical", "conservative"', - node.fundrawtransaction, rawtx, {"estimate_mode": mode, "conf_target": 0.1, "add_inputs": True}) + node.fundrawtransaction, rawtx, estimate_mode=mode, conf_target=0.1, add_inputs=True) self.log.info("Test fundrawtxn with invalid conf_target settings") for mode in ["unset", "economical", "conservative"]: self.log.debug("{}".format(mode)) for k, v in {"string": "", "object": {"foo": "bar"}}.items(): assert_raises_rpc_error(-3, f"JSON value of type {k} for field conf_target is not of expected type number", - node.fundrawtransaction, rawtx, {"estimate_mode": mode, "conf_target": v, "add_inputs": True}) + node.fundrawtransaction, rawtx, estimate_mode=mode, conf_target=v, add_inputs=True) for n in [-1, 0, 1009]: assert_raises_rpc_error(-8, "Invalid conf_target, must be between 1 and 1008", # max value of 1008 per src/policy/fees.h - node.fundrawtransaction, rawtx, {"estimate_mode": mode, "conf_target": n, "add_inputs": True}) + node.fundrawtransaction, rawtx, estimate_mode=mode, conf_target=n, add_inputs=True) self.log.info("Test invalid fee rate settings") for param, value in {("fee_rate", 100000), ("feeRate", 1.000)}: assert_raises_rpc_error(-4, "Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)", - node.fundrawtransaction, rawtx, {param: value, "add_inputs": True}) + node.fundrawtransaction, rawtx, add_inputs=True, **{param: value}) assert_raises_rpc_error(-3, "Amount out of range", - node.fundrawtransaction, rawtx, {param: -1, "add_inputs": True}) + node.fundrawtransaction, rawtx, add_inputs=True, **{param: -1}) assert_raises_rpc_error(-3, "Amount is not a number or string", - node.fundrawtransaction, rawtx, {param: {"foo": "bar"}, "add_inputs": True}) + node.fundrawtransaction, rawtx, add_inputs=True, **{param: {"foo": "bar"}}) # Test fee rate values that don't pass fixed-point parsing checks. for invalid_value in ["", 0.000000001, 1e-09, 1.111111111, 1111111111111111, "31.999999999999999999999"]: - assert_raises_rpc_error(-3, "Invalid amount", node.fundrawtransaction, rawtx, {param: invalid_value, "add_inputs": True}) + assert_raises_rpc_error(-3, "Invalid amount", node.fundrawtransaction, rawtx, add_inputs=True, **{param: invalid_value}) # Test fee_rate values that cannot be represented in sat/vB. for invalid_value in [0.0001, 0.00000001, 0.00099999, 31.99999999, "0.0001", "0.00000001", "0.00099999", "31.99999999"]: assert_raises_rpc_error(-3, "Invalid amount", - node.fundrawtransaction, rawtx, {"fee_rate": invalid_value, "add_inputs": True}) + node.fundrawtransaction, rawtx, fee_rate=invalid_value, add_inputs=True) self.log.info("Test min fee rate checks are bypassed with fundrawtxn, e.g. a fee_rate under 1 sat/vB is allowed") - node.fundrawtransaction(rawtx, {"fee_rate": 0.999, "add_inputs": True}) - node.fundrawtransaction(rawtx, {"feeRate": 0.00000999, "add_inputs": True}) + node.fundrawtransaction(rawtx, fee_rate=0.999, add_inputs=True) + node.fundrawtransaction(rawtx, feeRate=0.00000999, add_inputs=True) self.log.info("- raises RPC error if both feeRate and fee_rate are passed") assert_raises_rpc_error(-8, "Cannot specify both fee_rate (sat/vB) and feeRate (BTC/kvB)", - node.fundrawtransaction, rawtx, {"fee_rate": 0.1, "feeRate": 0.1, "add_inputs": True}) + node.fundrawtransaction, rawtx, fee_rate=0.1, feeRate=0.1, add_inputs=True) self.log.info("- raises RPC error if both feeRate and estimate_mode passed") assert_raises_rpc_error(-8, "Cannot specify both estimate_mode and feeRate", - node.fundrawtransaction, rawtx, {"estimate_mode": "economical", "feeRate": 0.1, "add_inputs": True}) + node.fundrawtransaction, rawtx, estimate_mode="economical", feeRate=0.1, add_inputs=True) for param in ["feeRate", "fee_rate"]: self.log.info("- raises RPC error if both {} and conf_target are passed".format(param)) @@ -854,7 +854,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.log.info("- raises RPC error if both fee_rate and estimate_mode are passed") assert_raises_rpc_error(-8, "Cannot specify both estimate_mode and fee_rate", - node.fundrawtransaction, rawtx, {"fee_rate": 1, "estimate_mode": "economical", "add_inputs": True}) + node.fundrawtransaction, rawtx, fee_rate=1, estimate_mode="economical", add_inputs=True) def test_address_reuse(self): """Test no address reuse occurs.""" @@ -884,10 +884,10 @@ class RawTransactionsTest(BitcoinTestFramework): # Test subtract fee from outputs with feeRate (BTC/kvB) result = [self.nodes[3].fundrawtransaction(rawtx), # uses self.min_relay_tx_fee (set by settxfee) - self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": []}), # empty subtraction list - self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": [0]}), # uses self.min_relay_tx_fee (set by settxfee) - self.nodes[3].fundrawtransaction(rawtx, {"feeRate": 2 * self.min_relay_tx_fee}), - self.nodes[3].fundrawtransaction(rawtx, {"feeRate": 2 * self.min_relay_tx_fee, "subtractFeeFromOutputs": [0]}),] + self.nodes[3].fundrawtransaction(rawtx, subtractFeeFromOutputs=[]), # empty subtraction list + self.nodes[3].fundrawtransaction(rawtx, subtractFeeFromOutputs=[0]), # uses self.min_relay_tx_fee (set by settxfee) + self.nodes[3].fundrawtransaction(rawtx, feeRate=2 * self.min_relay_tx_fee), + self.nodes[3].fundrawtransaction(rawtx, feeRate=2 * self.min_relay_tx_fee, subtractFeeFromOutputs=[0]),] dec_tx = [self.nodes[3].decoderawtransaction(tx_['hex']) for tx_ in result] output = [d['vout'][1 - r['changepos']]['value'] for d, r in zip(dec_tx, result)] change = [d['vout'][r['changepos']]['value'] for d, r in zip(dec_tx, result)] @@ -904,10 +904,10 @@ class RawTransactionsTest(BitcoinTestFramework): # Test subtract fee from outputs with fee_rate (sat/vB) btc_kvb_to_sat_vb = 100000 # (1e5) result = [self.nodes[3].fundrawtransaction(rawtx), # uses self.min_relay_tx_fee (set by settxfee) - self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": []}), # empty subtraction list - self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": [0]}), # uses self.min_relay_tx_fee (set by settxfee) - self.nodes[3].fundrawtransaction(rawtx, {"fee_rate": 2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee}), - self.nodes[3].fundrawtransaction(rawtx, {"fee_rate": 2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee, "subtractFeeFromOutputs": [0]}),] + self.nodes[3].fundrawtransaction(rawtx, subtractFeeFromOutputs=[]), # empty subtraction list + self.nodes[3].fundrawtransaction(rawtx, subtractFeeFromOutputs=[0]), # uses self.min_relay_tx_fee (set by settxfee) + self.nodes[3].fundrawtransaction(rawtx, fee_rate=2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee), + self.nodes[3].fundrawtransaction(rawtx, fee_rate=2 * btc_kvb_to_sat_vb * self.min_relay_tx_fee, subtractFeeFromOutputs=[0]),] dec_tx = [self.nodes[3].decoderawtransaction(tx_['hex']) for tx_ in result] output = [d['vout'][1 - r['changepos']]['value'] for d, r in zip(dec_tx, result)] change = [d['vout'][r['changepos']]['value'] for d, r in zip(dec_tx, result)] @@ -927,7 +927,7 @@ class RawTransactionsTest(BitcoinTestFramework): result = [self.nodes[3].fundrawtransaction(rawtx), # Split the fee between outputs 0, 2, and 3, but not output 1. - self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": [0, 2, 3]})] + self.nodes[3].fundrawtransaction(rawtx, subtractFeeFromOutputs=[0, 2, 3])] dec_tx = [self.nodes[3].decoderawtransaction(result[0]['hex']), self.nodes[3].decoderawtransaction(result[1]['hex'])] @@ -969,7 +969,7 @@ class RawTransactionsTest(BitcoinTestFramework): vout = find_vout_for_address(self.nodes[0], txid, addr) rawtx = self.nodes[0].createrawtransaction([{'txid': txid, 'vout': vout}], [{self.nodes[0].getnewaddress(): 5}]) - fundedtx = self.nodes[0].fundrawtransaction(rawtx, {'subtractFeeFromOutputs': [0]}) + fundedtx = self.nodes[0].fundrawtransaction(rawtx, subtractFeeFromOutputs=[0]) signedtx = self.nodes[0].signrawtransactionwithwallet(fundedtx['hex']) self.nodes[0].sendrawtransaction(signedtx['hex']) @@ -1027,25 +1027,25 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Not solvable pre-selected input COutPoint(%s, %s)" % (ext_utxo["txid"][0:10], ext_utxo["vout"]), wallet.fundrawtransaction, raw_tx) # Error conditions - assert_raises_rpc_error(-5, "'not a pubkey' is not hex", wallet.fundrawtransaction, raw_tx, {"solving_data": {"pubkeys":["not a pubkey"]}}) - assert_raises_rpc_error(-5, "'01234567890a0b0c0d0e0f' is not a valid public key", wallet.fundrawtransaction, raw_tx, {"solving_data": {"pubkeys":["01234567890a0b0c0d0e0f"]}}) - assert_raises_rpc_error(-5, "'not a script' is not hex", wallet.fundrawtransaction, raw_tx, {"solving_data": {"scripts":["not a script"]}}) - assert_raises_rpc_error(-8, "Unable to parse descriptor 'not a descriptor'", wallet.fundrawtransaction, raw_tx, {"solving_data": {"descriptors":["not a descriptor"]}}) - assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"]}]}) - assert_raises_rpc_error(-8, "Invalid parameter, vout cannot be negative", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": -1}]}) - assert_raises_rpc_error(-8, "Invalid parameter, missing weight key", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"]}]}) - assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be less than 165", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 164}]}) - assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be less than 165", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": -1}]}) - assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be greater than", wallet.fundrawtransaction, raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 400001}]}) + assert_raises_rpc_error(-5, "'not a pubkey' is not hex", wallet.fundrawtransaction, raw_tx, solving_data={"pubkeys":["not a pubkey"]}) + assert_raises_rpc_error(-5, "'01234567890a0b0c0d0e0f' is not a valid public key", wallet.fundrawtransaction, raw_tx, solving_data={"pubkeys":["01234567890a0b0c0d0e0f"]}) + assert_raises_rpc_error(-5, "'not a script' is not hex", wallet.fundrawtransaction, raw_tx, solving_data={"scripts":["not a script"]}) + assert_raises_rpc_error(-8, "Unable to parse descriptor 'not a descriptor'", wallet.fundrawtransaction, raw_tx, solving_data={"descriptors":["not a descriptor"]}) + assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"]}]) + assert_raises_rpc_error(-8, "Invalid parameter, vout cannot be negative", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": -1}]) + assert_raises_rpc_error(-8, "Invalid parameter, missing weight key", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"]}]) + assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be less than 165", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 164}]) + assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be less than 165", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": -1}]) + assert_raises_rpc_error(-8, "Invalid parameter, weight cannot be greater than", wallet.fundrawtransaction, raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 400001}]) # But funding should work when the solving data is provided - funded_tx = wallet.fundrawtransaction(raw_tx, {"solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + funded_tx = wallet.fundrawtransaction(raw_tx, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}) signed_tx = wallet.signrawtransactionwithwallet(funded_tx['hex']) assert not signed_tx['complete'] signed_tx = self.nodes[0].signrawtransactionwithwallet(signed_tx['hex']) assert signed_tx['complete'] - funded_tx = wallet.fundrawtransaction(raw_tx, {"solving_data": {"descriptors": [desc]}}) + funded_tx = wallet.fundrawtransaction(raw_tx, solving_data={"descriptors": [desc]}) signed_tx1 = wallet.signrawtransactionwithwallet(funded_tx['hex']) assert not signed_tx1['complete'] signed_tx2 = self.nodes[0].signrawtransactionwithwallet(signed_tx1['hex']) @@ -1060,30 +1060,30 @@ class RawTransactionsTest(BitcoinTestFramework): high_input_weight = input_weight * 2 # Funding should also work if the input weight is provided - funded_tx = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": input_weight}]}) + funded_tx = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": input_weight}]) signed_tx = wallet.signrawtransactionwithwallet(funded_tx["hex"]) signed_tx = self.nodes[0].signrawtransactionwithwallet(signed_tx["hex"]) assert_equal(self.nodes[0].testmempoolaccept([signed_tx["hex"]])[0]["allowed"], True) assert_equal(signed_tx["complete"], True) # Reducing the weight should have a lower fee - funded_tx2 = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": low_input_weight}]}) + funded_tx2 = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": low_input_weight}]) assert_greater_than(funded_tx["fee"], funded_tx2["fee"]) # Increasing the weight should have a higher fee - funded_tx2 = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}]}) + funded_tx2 = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}]) assert_greater_than(funded_tx2["fee"], funded_tx["fee"]) # The provided weight should override the calculated weight when solving data is provided - funded_tx3 = wallet.fundrawtransaction(raw_tx, {"solving_data": {"descriptors": [desc]}, "input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}]}) + funded_tx3 = wallet.fundrawtransaction(raw_tx, solving_data={"descriptors": [desc]}, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}]) assert_equal(funded_tx2["fee"], funded_tx3["fee"]) # The feerate should be met - funded_tx4 = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], "fee_rate": 10}) + funded_tx4 = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": high_input_weight}], fee_rate=10) input_add_weight = high_input_weight - (41 * 4) tx4_weight = wallet.decoderawtransaction(funded_tx4["hex"])["weight"] + input_add_weight tx4_vsize = int(ceil(tx4_weight / 4)) assert_fee_amount(funded_tx4["fee"], tx4_vsize, Decimal(0.0001)) # Funding with weight at csuint boundaries should not cause problems - funded_tx = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 255}]}) - funded_tx = wallet.fundrawtransaction(raw_tx, {"input_weights": [{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 65539}]}) + funded_tx = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 255}]) + funded_tx = wallet.fundrawtransaction(raw_tx, input_weights=[{"txid": ext_utxo["txid"], "vout": ext_utxo["vout"], "weight": 65539}]) self.nodes[2].unloadwallet("extfund") @@ -1123,7 +1123,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Fund wallet with 2 outputs, 5 BTC each. addr2 = wallet.getnewaddress(address_type="bech32") - source_tx = self.nodes[0].send(outputs=[{addr1: 5}, {addr2: 5}], options={"change_position": 0}) + source_tx = self.nodes[0].send(outputs=[{addr1: 5}, {addr2: 5}], change_position=0) self.generate(self.nodes[0], 1) # Select only one input. @@ -1135,12 +1135,12 @@ class RawTransactionsTest(BitcoinTestFramework): } ] } - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.send, outputs=[{addr1: 8}], options=options) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.send, outputs=[{addr1: 8}], **options) # Case (3), Explicit add_inputs=true and preset inputs (with preset inputs not-covering the target amount) options["add_inputs"] = True options["add_to_wallet"] = False - tx = wallet.send(outputs=[{addr1: 8}], options=options) + tx = wallet.send(outputs=[{addr1: 8}], **options) assert tx["complete"] # Case (4), Explicit add_inputs=true and preset inputs (with preset inputs covering the target amount) @@ -1148,7 +1148,7 @@ class RawTransactionsTest(BitcoinTestFramework): "txid": source_tx["txid"], "vout": 2 # change position was hardcoded to index 0 }) - tx = wallet.send(outputs=[{addr1: 8}], options=options) + tx = wallet.send(outputs=[{addr1: 8}], **options) assert tx["complete"] # Check that only the preset inputs were added to the tx decoded_psbt_inputs = self.nodes[0].decodepsbt(tx["psbt"])['tx']['vin'] @@ -1158,12 +1158,12 @@ class RawTransactionsTest(BitcoinTestFramework): # Case (5), assert that inputs are added to the tx by explicitly setting add_inputs=true options = {"add_inputs": True, "add_to_wallet": True} - tx = wallet.send(outputs=[{addr1: 8}], options=options) + tx = wallet.send(outputs=[{addr1: 8}], **options) assert tx["complete"] # 6. Explicit add_inputs=false, no preset inputs: options = {"add_inputs": False} - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.send, outputs=[{addr1: 3}], options=options) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.send, outputs=[{addr1: 3}], **options) ################################################ @@ -1184,14 +1184,14 @@ class RawTransactionsTest(BitcoinTestFramework): # Case (3), Explicit add_inputs=true and preset inputs (with preset inputs not-covering the target amount) options["add_inputs"] = True - assert "psbt" in wallet.walletcreatefundedpsbt(outputs=[{addr1: 8}], inputs=inputs, options=options) + assert "psbt" in wallet.walletcreatefundedpsbt(outputs=[{addr1: 8}], inputs=inputs, **options) # Case (4), Explicit add_inputs=true and preset inputs (with preset inputs covering the target amount) inputs.append({ "txid": source_tx["txid"], "vout": 2 # change position was hardcoded to index 0 }) - psbt_tx = wallet.walletcreatefundedpsbt(outputs=[{addr1: 8}], inputs=inputs, options=options) + psbt_tx = wallet.walletcreatefundedpsbt(outputs=[{addr1: 8}], inputs=inputs, **options) # Check that only the preset inputs were added to the tx decoded_psbt_inputs = self.nodes[0].decodepsbt(psbt_tx["psbt"])['tx']['vin'] assert_equal(len(decoded_psbt_inputs), 2) @@ -1203,11 +1203,11 @@ class RawTransactionsTest(BitcoinTestFramework): options = { "add_inputs": True } - assert "psbt" in wallet.walletcreatefundedpsbt(inputs=[], outputs=outputs, options=options) + assert "psbt" in wallet.walletcreatefundedpsbt(inputs=[], outputs=outputs, **options) # Case (6). Explicit add_inputs=false, no preset inputs: options = {"add_inputs": False} - assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.walletcreatefundedpsbt, inputs=[], outputs=outputs, options=options) + assert_raises_rpc_error(-4, ERR_NOT_ENOUGH_PRESET_INPUTS, wallet.walletcreatefundedpsbt, inputs=[], outputs=outputs, **options) self.nodes[2].unloadwallet("test_preset_inputs") @@ -1271,7 +1271,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.generate(self.nodes[0], 1) rawtx = wallet.createrawtransaction([{'txid': txid, 'vout': vout}], [{self.nodes[0].getnewaddress(address_type="bech32"): 8}]) - fundedtx = wallet.fundrawtransaction(rawtx, {'fee_rate': 10, "change_type": "bech32"}) + fundedtx = wallet.fundrawtransaction(rawtx, fee_rate=10, change_type="bech32") # with 71-byte signatures we should expect following tx size # tx overhead (10) + 2 inputs (41 each) + 2 p2wpkh (31 each) + (segwit marker and flag (2) + 2 p2wpkh 71 byte sig witnesses (107 each)) / witness scaling factor (4) tx_size = ceil(10 + 41*2 + 31*2 + (2 + 107*2)/4) @@ -1280,7 +1280,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Using the other output should have 72 byte sigs rawtx = wallet.createrawtransaction([{'txid': txid, 'vout': ext_vout}], [{self.nodes[0].getnewaddress(): 13}]) ext_desc = self.nodes[0].getaddressinfo(ext_addr)["desc"] - fundedtx = wallet.fundrawtransaction(rawtx, {'fee_rate': 10, "change_type": "bech32", "solving_data": {"descriptors": [ext_desc]}}) + fundedtx = wallet.fundrawtransaction(rawtx, fee_rate=10, change_type="bech32", solving_data={"descriptors": [ext_desc]}) # tx overhead (10) + 3 inputs (41 each) + 2 p2wpkh(31 each) + (segwit marker and flag (2) + 2 p2wpkh 71 bytes sig witnesses (107 each) + p2wpkh 72 byte sig witness (108)) / witness scaling factor (4) tx_size = ceil(10 + 41*3 + 31*2 + (2 + 107*2 + 108)/4) assert_equal(fundedtx['fee'] * COIN, tx_size * 10) @@ -1307,7 +1307,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Insufficient funds", wallet.fundrawtransaction, rawtx) # But we can opt-in to use them for funding. - fundedtx = wallet.fundrawtransaction(rawtx, {"include_unsafe": True}) + fundedtx = wallet.fundrawtransaction(rawtx, include_unsafe=True) tx_dec = wallet.decoderawtransaction(fundedtx['hex']) assert all((txin["txid"], txin["vout"]) in inputs for txin in tx_dec["vin"]) signedtx = wallet.signrawtransactionwithwallet(fundedtx['hex']) @@ -1315,7 +1315,7 @@ class RawTransactionsTest(BitcoinTestFramework): # And we can also use them once they're confirmed. self.generate(self.nodes[0], 1) - fundedtx = wallet.fundrawtransaction(rawtx, {"include_unsafe": False}) + fundedtx = wallet.fundrawtransaction(rawtx, include_unsafe=False) tx_dec = wallet.decoderawtransaction(fundedtx['hex']) assert all((txin["txid"], txin["vout"]) in inputs for txin in tx_dec["vin"]) signedtx = wallet.signrawtransactionwithwallet(fundedtx['hex']) @@ -1350,7 +1350,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Create transactions in order to calculate fees for the target bounds that can trigger this bug change_tx = tester.fundrawtransaction(tester.createrawtransaction([], [{funds.getnewaddress(): 1.5}])) tx = tester.createrawtransaction([], [{funds.getnewaddress(): 2}]) - no_change_tx = tester.fundrawtransaction(tx, {"subtractFeeFromOutputs": [0]}) + no_change_tx = tester.fundrawtransaction(tx, subtractFeeFromOutputs=[0]) overhead_fees = feerate * len(tx) / 2 / 1000 cost_of_change = change_tx["fee"] - no_change_tx["fee"] @@ -1402,7 +1402,7 @@ class RawTransactionsTest(BitcoinTestFramework): # To test this does not happen, we subtract 202 sats from the input value. If working correctly, this should # fail with insufficient funds rather than bitcoind asserting. rawtx = w.createrawtransaction(inputs=[], outputs=[{self.nodes[0].getnewaddress(address_type="bech32"): 1 - 0.00000202}]) - assert_raises_rpc_error(-4, "Insufficient funds", w.fundrawtransaction, rawtx, {"fee_rate": 1.85}) + assert_raises_rpc_error(-4, "Insufficient funds", w.fundrawtransaction, rawtx, fee_rate=1.85) def test_input_confs_control(self): self.nodes[0].createwallet("minconf") diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 0f79df6e5d..8f84d8ed60 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -41,7 +41,7 @@ class WalletHDTest(BitcoinTestFramework): change_addr = self.nodes[1].getrawchangeaddress() change_addrV = self.nodes[1].getaddressinfo(change_addr) if self.options.descriptors: - assert_equal(change_addrV["hdkeypath"], "m/84'/1'/0'/1/0") + assert_equal(change_addrV["hdkeypath"], "m/84h/1h/0h/1/0") else: assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key @@ -63,7 +63,7 @@ class WalletHDTest(BitcoinTestFramework): hd_add = self.nodes[1].getnewaddress() hd_info = self.nodes[1].getaddressinfo(hd_add) if self.options.descriptors: - assert_equal(hd_info["hdkeypath"], "m/84'/1'/0'/0/" + str(i)) + assert_equal(hd_info["hdkeypath"], "m/84h/1h/0h/0/" + str(i)) else: assert_equal(hd_info["hdkeypath"], "m/0'/0'/" + str(i) + "'") assert_equal(hd_info["hdmasterfingerprint"], hd_fingerprint) @@ -76,7 +76,7 @@ class WalletHDTest(BitcoinTestFramework): change_addr = self.nodes[1].getrawchangeaddress() change_addrV = self.nodes[1].getaddressinfo(change_addr) if self.options.descriptors: - assert_equal(change_addrV["hdkeypath"], "m/84'/1'/0'/1/1") + assert_equal(change_addrV["hdkeypath"], "m/84h/1h/0h/1/1") else: assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key @@ -101,7 +101,7 @@ class WalletHDTest(BitcoinTestFramework): hd_add_2 = self.nodes[1].getnewaddress() hd_info_2 = self.nodes[1].getaddressinfo(hd_add_2) if self.options.descriptors: - assert_equal(hd_info_2["hdkeypath"], "m/84'/1'/0'/0/" + str(i)) + assert_equal(hd_info_2["hdkeypath"], "m/84h/1h/0h/0/" + str(i)) else: assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/" + str(i) + "'") assert_equal(hd_info_2["hdmasterfingerprint"], hd_fingerprint) @@ -143,7 +143,7 @@ class WalletHDTest(BitcoinTestFramework): keypath = self.nodes[1].getaddressinfo(out['scriptPubKey']['address'])['hdkeypath'] if self.options.descriptors: - assert_equal(keypath[0:14], "m/84'/1'/0'/1/") + assert_equal(keypath[0:14], "m/84h/1h/0h/1/") else: assert_equal(keypath[0:7], "m/0'/1'") diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index 211e939a39..0ac67607e1 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -75,7 +75,7 @@ class Variant(collections.namedtuple("Variant", "call data address_type rescan p request.update({"redeemscript": self.address['embedded']['scriptPubKey']}) response = self.node.importmulti( requests=[request], - options={"rescan": self.rescan in (Rescan.yes, Rescan.late_timestamp)}, + rescan=self.rescan in (Rescan.yes, Rescan.late_timestamp), ) assert_equal(response, [{"success": True}]) diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index 4f2db2018a..ad5ae111aa 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -332,15 +332,15 @@ class ImportDescriptorsTest(BitcoinTestFramework): assert_raises_rpc_error(-4, 'This wallet has no available keys', w1.getrawchangeaddress, 'bech32') assert_equal(received_addr, expected_addr) bech32_addr_info = w1.getaddressinfo(received_addr) - assert_equal(bech32_addr_info['desc'][:23], 'wpkh([80002067/0\'/0\'/{}]'.format(i)) + assert_equal(bech32_addr_info['desc'][:23], 'wpkh([80002067/0h/0h/{}]'.format(i)) shwpkh_addr = w1.getnewaddress('', 'p2sh-segwit') shwpkh_addr_info = w1.getaddressinfo(shwpkh_addr) - assert_equal(shwpkh_addr_info['desc'][:26], 'sh(wpkh([abcdef12/0\'/0\'/{}]'.format(i)) + assert_equal(shwpkh_addr_info['desc'][:26], 'sh(wpkh([abcdef12/0h/0h/{}]'.format(i)) pkh_addr = w1.getnewaddress('', 'legacy') pkh_addr_info = w1.getaddressinfo(pkh_addr) - assert_equal(pkh_addr_info['desc'][:22], 'pkh([12345678/0\'/0\'/{}]'.format(i)) + assert_equal(pkh_addr_info['desc'][:22], 'pkh([12345678/0h/0h/{}]'.format(i)) assert_equal(w1.getwalletinfo()['keypoolsize'], 4 * 3) # After retrieving a key, we don't refill the keypool again, so it's one less for each address type w1.keypoolrefill() diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index bd97851153..a39db3bfb8 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -178,30 +178,30 @@ class KeyPoolTest(BitcoinTestFramework): # Using a fee rate (10 sat / byte) well above the minimum relay rate # creating a 5,000 sat transaction with change should not be possible - assert_raises_rpc_error(-4, "Transaction needs a change address, but we can't generate it.", w2.walletcreatefundedpsbt, inputs=[], outputs=[{addr.pop(): 0.00005000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + assert_raises_rpc_error(-4, "Transaction needs a change address, but we can't generate it.", w2.walletcreatefundedpsbt, inputs=[], outputs=[{addr.pop(): 0.00005000}], subtractFeeFromOutputs=[0], feeRate=0.00010) # creating a 10,000 sat transaction without change, with a manual input, should still be possible - res = w2.walletcreatefundedpsbt(inputs=w2.listunspent(), outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + res = w2.walletcreatefundedpsbt(inputs=w2.listunspent(), outputs=[{destination: 0.00010000}], subtractFeeFromOutputs=[0], feeRate=0.00010) assert_equal("psbt" in res, True) # creating a 10,000 sat transaction without change should still be possible - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], subtractFeeFromOutputs=[0], feeRate=0.00010) assert_equal("psbt" in res, True) # should work without subtractFeeFromOutputs if the exact fee is subtracted from the amount - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008900}], options={"feeRate": 0.00010}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008900}], feeRate=0.00010) assert_equal("psbt" in res, True) # dust change should be removed - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008800}], options={"feeRate": 0.00010}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008800}], feeRate=0.00010) assert_equal("psbt" in res, True) # create a transaction without change at the maximum fee rate, such that the output is still spendable: - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.0008823}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], subtractFeeFromOutputs=[0], feeRate=0.0008823) assert_equal("psbt" in res, True) assert_equal(res["fee"], Decimal("0.00009706")) # creating a 10,000 sat transaction with a manual change address should be possible - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010, "changeAddress": addr.pop()}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], subtractFeeFromOutputs=[0], feeRate=0.00010, changeAddress=addr.pop()) assert_equal("psbt" in res, True) if not self.options.descriptors: diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index 18c3ae1f7c..f1458bb374 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -83,11 +83,11 @@ class KeypoolRestoreTest(BitcoinTestFramework): # Check that we have marked all keys up to the used keypool key as used if self.options.descriptors: if output_type == 'legacy': - assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/44'/1'/0'/0/110") + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/44h/1h/0h/0/110") elif output_type == 'p2sh-segwit': - assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/49'/1'/0'/0/110") + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/49h/1h/0h/0/110") elif output_type == 'bech32': - assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/84'/1'/0'/0/110") + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/84h/1h/0h/0/110") else: assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/0'/0'/110'") diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py index c5479089c6..712b881322 100755 --- a/test/functional/wallet_listdescriptors.py +++ b/test/functional/wallet_listdescriptors.py @@ -65,7 +65,7 @@ class ListDescriptorsTest(BitcoinTestFramework): self.log.info('Test descriptors with hardened derivations are listed in importable form.') xprv = 'tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg' xpub_acc = 'tpubDCMVLhErorrAGfApiJSJzEKwqeaf2z3NrkVMxgYQjZLzMjXMBeRw2muGNYbvaekAE8rUFLftyEar4LdrG2wXyyTJQZ26zptmeTEjPTaATts' - hardened_path = '/84\'/1\'/0\'' + hardened_path = '/84h/1h/0h' wallet = node.get_wallet_rpc('w2') wallet.importdescriptors([{ 'desc': descsum_create('wpkh(' + xprv + hardened_path + '/0/*)'), diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index 7c2959bb89..c6c2af10b1 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -53,7 +53,7 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(addr_info["address"], addr_info_old["address"]) assert_equal(addr_info["scriptPubKey"], addr_info_old["scriptPubKey"]) assert_equal(addr_info["ismine"], addr_info_old["ismine"]) - assert_equal(addr_info["hdkeypath"], addr_info_old["hdkeypath"]) + assert_equal(addr_info["hdkeypath"], addr_info_old["hdkeypath"].replace("'","h")) assert_equal(addr_info["solvable"], addr_info_old["solvable"]) assert_equal(addr_info["ischange"], addr_info_old["ischange"]) assert_equal(addr_info["hdmasterfingerprint"], addr_info_old["hdmasterfingerprint"]) @@ -91,11 +91,11 @@ class WalletMigrationTest(BitcoinTestFramework): self.assert_is_sqlite("basic0") # The wallet should create the following descriptors: - # * BIP32 descriptors in the form of "0'/0'/*" and "0'/1'/*" (2 descriptors) - # * BIP44 descriptors in the form of "44'/1'/0'/0/*" and "44'/1'/0'/1/*" (2 descriptors) - # * BIP49 descriptors, P2SH(P2WPKH), in the form of "86'/1'/0'/0/*" and "86'/1'/0'/1/*" (2 descriptors) - # * BIP84 descriptors, P2WPKH, in the form of "84'/1'/0'/1/*" and "84'/1'/0'/1/*" (2 descriptors) - # * BIP86 descriptors, P2TR, in the form of "86'/1'/0'/0/*" and "86'/1'/0'/1/*" (2 descriptors) + # * BIP32 descriptors in the form of "0h/0h/*" and "0h/1h/*" (2 descriptors) + # * BIP44 descriptors in the form of "44h/1h/0h/0/*" and "44h/1h/0h/1/*" (2 descriptors) + # * BIP49 descriptors, P2SH(P2WPKH), in the form of "86h/1h/0h/0/*" and "86h/1h/0h/1/*" (2 descriptors) + # * BIP84 descriptors, P2WPKH, in the form of "84h/1h/0h/1/*" and "84h/1h/0h/1/*" (2 descriptors) + # * BIP86 descriptors, P2TR, in the form of "86h/1h/0h/0/*" and "86h/1h/0h/1/*" (2 descriptors) # * A combo(PK) descriptor for the wallet master key. # So, should have a total of 11 descriptors on it. assert_equal(len(basic0.listdescriptors()["descriptors"]), 11) @@ -107,7 +107,7 @@ class WalletMigrationTest(BitcoinTestFramework): self.assert_addr_info_equal(change_addr_info, old_change_addr_info) addr_info = basic0.getaddressinfo(basic0.getnewaddress("", "bech32")) - assert_equal(addr_info["hdkeypath"], "m/84'/1'/0'/0/0") + assert_equal(addr_info["hdkeypath"], "m/84h/1h/0h/0/0") self.log.info("Test migration of a basic keys only wallet with a balance") basic1 = self.create_legacy_wallet("basic1") @@ -281,7 +281,7 @@ class WalletMigrationTest(BitcoinTestFramework): imports0.importaddress(import_sent_addr) received_sent_watchonly_txid = default.sendtoaddress(import_sent_addr, 10) received_sent_watchonly_vout = find_vout_for_address(self.nodes[0], received_sent_watchonly_txid, import_sent_addr) - send = default.sendall(recipients=[default.getnewaddress()], options={"inputs": [{"txid": received_sent_watchonly_txid, "vout": received_sent_watchonly_vout}]}) + send = default.sendall(recipients=[default.getnewaddress()], inputs=[{"txid": received_sent_watchonly_txid, "vout": received_sent_watchonly_vout}]) sent_watchonly_txid = send["txid"] self.generate(self.nodes[0], 1) diff --git a/test/functional/wallet_multisig_descriptor_psbt.py b/test/functional/wallet_multisig_descriptor_psbt.py index 12069fb00d..28bee1911e 100755 --- a/test/functional/wallet_multisig_descriptor_psbt.py +++ b/test/functional/wallet_multisig_descriptor_psbt.py @@ -121,7 +121,7 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): to = participants["signers"][self.N - 1].getnewaddress() value = 1 self.log.info("First, make a sending transaction, created using `walletcreatefundedpsbt` (anyone can initiate this)...") - psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, options={"feeRate": 0.00010}) + psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, feeRate=0.00010) psbts = [] self.log.info("Now at least M users check the psbt with decodepsbt and (if OK) signs it with walletprocesspsbt...") @@ -143,7 +143,7 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): assert_equal(participants["signers"][self.N - 1].getbalance(), value) self.log.info("Send another transaction from the multisig, this time with a daisy chained signing flow (one after another in series)!") - psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, options={"feeRate": 0.00010}) + psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, feeRate=0.00010) for m in range(self.M): signers_multisig = participants["multisigs"][m] self._check_psbt(psbt["psbt"], to, value, signers_multisig) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 7e4a4002b2..7bdb6f5e3a 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -35,7 +35,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): self.log.info("Create a new transaction and wait until it's broadcast") parent_utxo, indep_utxo = node.listunspent()[:2] addr = node.getnewaddress() - txid = node.send(outputs=[{addr: 1}], options={"inputs": [parent_utxo]})["txid"] + txid = node.send(outputs=[{addr: 1}], inputs=[parent_utxo])["txid"] # Can take a few seconds due to transaction trickling peer_first.wait_for_broadcast([txid]) @@ -86,7 +86,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): # ordering of mapWallet is, if the child is not before the parent, we will create a new # child (via bumpfee) and remove the old child (via removeprunedfunds) until we get the # ordering of child before parent. - child_txid = node.send(outputs=[{addr: 0.5}], options={"inputs": [{"txid":txid, "vout":0}]})["txid"] + child_txid = node.send(outputs=[{addr: 0.5}], inputs=[{"txid":txid, "vout":0}])["txid"] while True: txids = node.listreceivedbyaddress(minconf=0, address_filter=addr)[0]["txids"] if txids == [child_txid, txid]: @@ -111,7 +111,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): # Evict these txs from the mempool evict_time = block_time + 60 * 60 * DEFAULT_MEMPOOL_EXPIRY_HOURS + 5 node.setmocktime(evict_time) - indep_send = node.send(outputs=[{node.getnewaddress(): 1}], options={"inputs": [indep_utxo]}) + indep_send = node.send(outputs=[{node.getnewaddress(): 1}], inputs=[indep_utxo]) node.getmempoolentry(indep_send["txid"]) assert_raises_rpc_error(-5, "Transaction not in mempool", node.getmempoolentry, txid) assert_raises_rpc_error(-5, "Transaction not in mempool", node.getmempoolentry, child_txid) diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py index f6440f07d7..c2b800df21 100755 --- a/test/functional/wallet_sendall.py +++ b/test/functional/wallet_sendall.py @@ -151,7 +151,7 @@ class SendallTest(BitcoinTestFramework): self.log.info("Test sending more than balance") pre_sendall_balance = self.add_utxos([7, 14]) - expected_tx = self.wallet.sendall(recipients=[{self.recipient: 5}, self.remainder_target], options={"add_to_wallet": False}) + expected_tx = self.wallet.sendall(recipients=[{self.recipient: 5}, self.remainder_target], add_to_wallet=False) tx = self.wallet.decoderawtransaction(expected_tx['hex']) fee = 21 - sum([o["value"] for o in tx["vout"]]) @@ -190,7 +190,7 @@ class SendallTest(BitcoinTestFramework): self.add_utxos([0.00000400, 0.00000300, 1]) # sendall with send_max - sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, options={"send_max": True}) + sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], fee_rate=300, send_max=True) tx_from_wallet = self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) assert_equal(len(tx_from_wallet["decoded"]["vin"]), 1) @@ -206,7 +206,7 @@ class SendallTest(BitcoinTestFramework): self.add_utxos([17, 4]) utxo = self.wallet.listunspent()[0] - sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], options={"inputs": [utxo]}) + sendall_tx_receipt = self.wallet.sendall(recipients=[self.remainder_target], inputs=[utxo]) tx_from_wallet = self.wallet.gettransaction(txid = sendall_tx_receipt["txid"], verbose = True) assert_equal(len(tx_from_wallet["decoded"]["vin"]), 1) assert_equal(len(tx_from_wallet["decoded"]["vout"]), 1) @@ -227,26 +227,26 @@ class SendallTest(BitcoinTestFramework): # fails on out of bounds vout assert_raises_rpc_error(-8, "Input not found. UTXO ({}:{}) is not part of wallet.".format(spent_utxo["txid"], 1000), - self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [{"txid": spent_utxo["txid"], "vout": 1000}]}) + self.wallet.sendall, recipients=[self.remainder_target], inputs=[{"txid": spent_utxo["txid"], "vout": 1000}]) # fails on unconfirmed spent UTXO self.wallet.sendall(recipients=[self.remainder_target]) assert_raises_rpc_error(-8, "Input not available. UTXO ({}:{}) was already spent.".format(spent_utxo["txid"], spent_utxo["vout"]), - self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [spent_utxo]}) + self.wallet.sendall, recipients=[self.remainder_target], inputs=[spent_utxo]) # fails on specific previously spent UTXO, while other UTXOs exist self.generate(self.nodes[0], 1) self.add_utxos([19, 2]) assert_raises_rpc_error(-8, "Input not available. UTXO ({}:{}) was already spent.".format(spent_utxo["txid"], spent_utxo["vout"]), - self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [spent_utxo]}) + self.wallet.sendall, recipients=[self.remainder_target], inputs=[spent_utxo]) # fails because UTXO is unknown, while other UTXOs exist foreign_utxo = self.def_wallet.listunspent()[0] assert_raises_rpc_error(-8, "Input not found. UTXO ({}:{}) is not part of wallet.".format(foreign_utxo["txid"], foreign_utxo["vout"]), self.wallet.sendall, recipients=[self.remainder_target], - options={"inputs": [foreign_utxo]}) + inputs=[foreign_utxo]) @cleanup def sendall_fails_on_no_address(self): @@ -270,7 +270,7 @@ class SendallTest(BitcoinTestFramework): "Cannot combine send_max with specific inputs.", self.wallet.sendall, recipients=[self.remainder_target], - options={"inputs": [utxo], "send_max": True}) + inputs=[utxo], send_max=True) @cleanup def sendall_fails_on_high_fee(self): @@ -308,7 +308,7 @@ class SendallTest(BitcoinTestFramework): else: watchonly.importmulti(import_req) - sendall_tx_receipt = watchonly.sendall(recipients=[self.remainder_target], options={"inputs": [utxo]}) + sendall_tx_receipt = watchonly.sendall(recipients=[self.remainder_target], inputs=[utxo]) psbt = sendall_tx_receipt["psbt"] decoded = self.nodes[0].decodepsbt(psbt) assert_equal(len(decoded["inputs"]), 1) diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 8d25044e43..c414147c65 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -109,28 +109,28 @@ class WalletSignerTest(BitcoinTestFramework): address_info = hww.getaddressinfo(address1) assert_equal(address_info['solvable'], True) assert_equal(address_info['ismine'], True) - assert_equal(address_info['hdkeypath'], "m/84'/1'/0'/0/0") + assert_equal(address_info['hdkeypath'], "m/84h/1h/0h/0/0") address2 = hww.getnewaddress(address_type="p2sh-segwit") assert_equal(address2, "2N2gQKzjUe47gM8p1JZxaAkTcoHPXV6YyVp") address_info = hww.getaddressinfo(address2) assert_equal(address_info['solvable'], True) assert_equal(address_info['ismine'], True) - assert_equal(address_info['hdkeypath'], "m/49'/1'/0'/0/0") + assert_equal(address_info['hdkeypath'], "m/49h/1h/0h/0/0") address3 = hww.getnewaddress(address_type="legacy") assert_equal(address3, "n1LKejAadN6hg2FrBXoU1KrwX4uK16mco9") address_info = hww.getaddressinfo(address3) assert_equal(address_info['solvable'], True) assert_equal(address_info['ismine'], True) - assert_equal(address_info['hdkeypath'], "m/44'/1'/0'/0/0") + assert_equal(address_info['hdkeypath'], "m/44h/1h/0h/0/0") address4 = hww.getnewaddress(address_type="bech32m") assert_equal(address4, "bcrt1phw4cgpt6cd30kz9k4wkpwm872cdvhss29jga2xpmftelhqll62ms4e9sqj") address_info = hww.getaddressinfo(address4) assert_equal(address_info['solvable'], True) assert_equal(address_info['ismine'], True) - assert_equal(address_info['hdkeypath'], "m/86'/1'/0'/0/0") + assert_equal(address_info['hdkeypath'], "m/86h/1h/0h/0/0") self.log.info('Test walletdisplayaddress') result = hww.walletdisplayaddress(address1) @@ -153,14 +153,14 @@ class WalletSignerTest(BitcoinTestFramework): assert mock_wallet.getwalletinfo()['private_keys_enabled'] result = mock_wallet.importdescriptors([{ - "desc": "tr([00000001/86'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#0jtt2jc9", + "desc": "tr([00000001/86h/1h/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#7ew68cn8", "timestamp": 0, "range": [0,1], "internal": False, "active": True }, { - "desc": "tr([00000001/86'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#7xw2h8ga", + "desc": "tr([00000001/86h/1h/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#0dtm6drl", "timestamp": 0, "range": [0, 0], "internal": True, @@ -182,7 +182,7 @@ class WalletSignerTest(BitcoinTestFramework): # hww4 = self.nodes[1].get_wallet_rpc("hww4") # # descriptors = [{ - # "desc": "wpkh([00000001/84'/1'/0']tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/0/*)#x30uthjs", + # "desc": "wpkh([00000001/84h/1h/0']tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/0/*)#x30uthjs", # "timestamp": "now", # "range": [0, 1], # "internal": False, @@ -190,7 +190,7 @@ class WalletSignerTest(BitcoinTestFramework): # "active": True # }, # { - # "desc": "wpkh([00000001/84'/1'/0']tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/*)#h92akzzg", + # "desc": "wpkh([00000001/84h/1h/0']tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/*)#h92akzzg", # "timestamp": "now", # "range": [0, 0], # "internal": True, @@ -211,13 +211,13 @@ class WalletSignerTest(BitcoinTestFramework): self.log.info('Test send using hww1') # Don't broadcast transaction yet so the RPC returns the raw hex - res = hww.send(outputs={dest:0.5},options={"add_to_wallet": False}) + res = hww.send(outputs={dest:0.5},add_to_wallet=False) assert res["complete"] assert_equal(res["hex"], mock_tx) self.log.info('Test sendall using hww1') - res = hww.sendall(recipients=[{dest:0.5}, hww.getrawchangeaddress()],options={"add_to_wallet": False}) + res = hww.sendall(recipients=[{dest:0.5}, hww.getrawchangeaddress()], add_to_wallet=False) assert res["complete"] assert_equal(res["hex"], mock_tx) # Broadcast transaction so we can bump the fee diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index b52892704f..a5d7445ce8 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -378,7 +378,7 @@ class WalletTaprootTest(BitcoinTestFramework): assert psbt_online.gettransaction(txid)['confirmations'] > 0 # Cleanup - psbt = psbt_online.sendall(recipients=[self.boring.getnewaddress()], options={"psbt": True})["psbt"] + psbt = psbt_online.sendall(recipients=[self.boring.getnewaddress()], psbt=True)["psbt"] res = psbt_offline.walletprocesspsbt(psbt=psbt, finalize=False) rawtx = self.nodes[0].finalizepsbt(res['psbt'])['hex'] txid = self.nodes[0].sendrawtransaction(rawtx) diff --git a/test/functional/wallet_timelock.py b/test/functional/wallet_timelock.py index 57a7c3907a..0a622979a4 100755 --- a/test/functional/wallet_timelock.py +++ b/test/functional/wallet_timelock.py @@ -29,7 +29,7 @@ class WalletLocktimeTest(BitcoinTestFramework): self.log.info("Send to new address with locktime") node.send( outputs={address: 5}, - options={"locktime": mtp_tip - 1}, + locktime=mtp_tip - 1, ) self.generate(node, 1) diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index dd4514318c..b473f5d65c 100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py @@ -98,13 +98,13 @@ class CreateWalletWatchonlyTest(BitcoinTestFramework): options = {'changeAddress': wo_change} no_wo_options = {'changeAddress': wo_change, 'includeWatching': False} - result = wo_wallet.walletcreatefundedpsbt(inputs=inputs, outputs=outputs, options=options) + result = wo_wallet.walletcreatefundedpsbt(inputs=inputs, outputs=outputs, **options) assert_equal("psbt" in result, True) assert_raises_rpc_error(-4, "Insufficient funds", wo_wallet.walletcreatefundedpsbt, inputs, outputs, 0, no_wo_options) self.log.info('Testing fundrawtransaction watch-only defaults') rawtx = wo_wallet.createrawtransaction(inputs=inputs, outputs=outputs) - result = wo_wallet.fundrawtransaction(hexstring=rawtx, options=options) + result = wo_wallet.fundrawtransaction(hexstring=rawtx, **options) assert_equal("hex" in result, True) assert_raises_rpc_error(-4, "Insufficient funds", wo_wallet.fundrawtransaction, rawtx, no_wo_options) diff --git a/test/lint/lint-includes.py b/test/lint/lint-includes.py index 459030bb0b..b14caa4855 100755 --- a/test/lint/lint-includes.py +++ b/test/lint/lint-includes.py @@ -23,15 +23,20 @@ EXCLUDED_DIRS = ["src/leveldb/", EXPECTED_BOOST_INCLUDES = ["boost/date_time/posix_time/posix_time.hpp", "boost/multi_index/hashed_index.hpp", + "boost/multi_index/identity.hpp", + "boost/multi_index/indexed_by.hpp", "boost/multi_index/ordered_index.hpp", "boost/multi_index/sequenced_index.hpp", + "boost/multi_index/tag.hpp", "boost/multi_index_container.hpp", "boost/process.hpp", "boost/signals2/connection.hpp", "boost/signals2/optional_last_value.hpp", "boost/signals2/signal.hpp", "boost/test/included/unit_test.hpp", - "boost/test/unit_test.hpp"] + "boost/test/unit_test.hpp", + "boost/tuple/tuple.hpp", + ] def get_toplevel(): diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py index 9de13e44e2..539d0acb5d 100755 --- a/test/lint/lint-python.py +++ b/test/lint/lint-python.py @@ -13,7 +13,7 @@ import pkg_resources import subprocess import sys -DEPS = ['flake8', 'mypy', 'pyzmq'] +DEPS = ['flake8', 'lief', 'mypy', 'pyzmq'] MYPY_CACHE_DIR = f"{os.getenv('BASE_ROOT_DIR', '')}/test/.mypy_cache" # All .py files, except those in src/ (to exclude subtrees there) diff --git a/test/lint/run-lint-format-strings.py b/test/lint/run-lint-format-strings.py index 91915f05f9..d1896dba84 100755 --- a/test/lint/run-lint-format-strings.py +++ b/test/lint/run-lint-format-strings.py @@ -241,20 +241,32 @@ def count_format_specifiers(format_string): 3 >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") 4 + >>> count_format_specifiers("foo %5$d") + 5 + >>> count_format_specifiers("foo %5$*7$d") + 7 """ assert type(format_string) is str format_string = format_string.replace('%%', 'X') - n = 0 - in_specifier = False - for i, char in enumerate(format_string): - if char == "%": - in_specifier = True + n = max_pos = 0 + for m in re.finditer("%(.*?)[aAcdeEfFgGinopsuxX]", format_string, re.DOTALL): + # Increase the max position if the argument has a position number like + # "5$", otherwise increment the argument count. + pos_num, = re.match(r"(?:(^\d+)\$)?", m.group(1)).groups() + if pos_num is not None: + max_pos = max(max_pos, int(pos_num)) + else: n += 1 - elif char in "aAcdeEfFgGinopsuxX": - in_specifier = False - elif in_specifier and char == "*": + + # Increase the max position if there is a "*" width argument with a + # position like "*7$", and increment the argument count if there is a + # "*" width argument with no position. + star, star_pos_num = re.match(r"(?:.*?(\*(?:(\d+)\$)?)|)", m.group(1)).groups() + if star_pos_num is not None: + max_pos = max(max_pos, int(star_pos_num)) + elif star is not None: n += 1 - return n + return max(n, max_pos) def main(): diff --git a/test/lint/spelling.ignore-words.txt b/test/lint/spelling.ignore-words.txt index d44dd70684..fa47a31725 100644 --- a/test/lint/spelling.ignore-words.txt +++ b/test/lint/spelling.ignore-words.txt @@ -11,6 +11,7 @@ hights inflight invokable keypair +lief mor nd nin diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 5a790b72f2..82c6885bc4 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -25,6 +25,7 @@ race:src/qt/test/* deadlock:src/qt/test/* # External libraries +# https://github.com/bitcoin/bitcoin/pull/27658#issuecomment-1547639621 deadlock:libdb race:libzmq diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 6b891c462e..ae70fb49d4 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -56,7 +56,6 @@ implicit-integer-sign-change:prevector.h implicit-integer-sign-change:script/bitcoinconsensus.cpp implicit-integer-sign-change:script/interpreter.cpp implicit-integer-sign-change:serialize.h -implicit-integer-sign-change:txmempool.cpp implicit-signed-integer-truncation:crypto/ implicit-unsigned-integer-truncation:crypto/ shift-base:arith_uint256.cpp diff --git a/test/util/test_runner.py b/test/util/test_runner.py index e5cdd0bc3a..1cd368f6f4 100755 --- a/test/util/test_runner.py +++ b/test/util/test_runner.py @@ -74,6 +74,11 @@ def bctest(testDir, testObj, buildenv): """ # Get the exec names and arguments execprog = os.path.join(buildenv["BUILDDIR"], "src", testObj["exec"] + buildenv["EXEEXT"]) + if testObj["exec"] == "./bitcoin-util": + execprog = os.getenv("BITCOINUTIL", default=execprog) + elif testObj["exec"] == "./bitcoin-tx": + execprog = os.getenv("BITCOINTX", default=execprog) + execargs = testObj['args'] execrun = [execprog] + execargs |