diff options
233 files changed, 5185 insertions, 1988 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 6dc029ee51..01e0cebc92 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -112,7 +112,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" task: - name: '[previous releases, uses qt5 dev package and some depends packages] [unsigned char] [bionic]' + name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [bionic]' previous_releases_cache: folder: "releases" << : *GLOBAL_TASK_TEMPLATE @@ -143,10 +143,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" task: - name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [focal]' + name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [hirsute]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:focal + image: ubuntu:hirsute env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" @@ -161,7 +161,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: '[multiprocess] [focal]' + name: '[multiprocess, DEBUG] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal diff --git a/.fuzzbuzz.yml b/.fuzzbuzz.yml deleted file mode 100644 index e40b4df165..0000000000 --- a/.fuzzbuzz.yml +++ /dev/null @@ -1,16 +0,0 @@ -base: ubuntu:16.04 -language: c++ -engine: libFuzzer -environment: - - CXXFLAGS=-fcoverage-mapping -fno-omit-frame-pointer -fprofile-instr-generate -gline-tables-only -O1 -setup: - - sudo apt-get update - - sudo apt-get install -y autoconf bsdmainutils clang git libboost-system-dev libboost-filesystem-dev libboost-test-dev libc++1 libc++abi1 libc++abi-dev libc++-dev libclang1 libclang-dev libdb5.3++ libevent-dev libllvm-ocaml-dev libomp5 libomp-dev libqt5core5a libqt5dbus5 libqt5gui5 libtool llvm llvm-dev llvm-runtime pkg-config qttools5-dev qttools5-dev-tools software-properties-common - - ./autogen.sh - - CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined --enable-danger-fuzz-link-all - - make - - git clone https://github.com/bitcoin-core/qa-assets -auto_targets: - find_targets_command: find src/test/fuzz/ -executable -type f ! -name "*.cpp" ! -name "*.h" - base_corpus_dir: qa-assets/fuzz_seed_corpus/ - memory_limit: none @@ -19,6 +19,7 @@ # Maintainers # @fanquake +# @hebasto # @jonasschnelli # @laanwj # @marcofalke @@ -59,11 +60,16 @@ /src/test/fuzz/ @practicalswift /doc/fuzzing.md @practicalswift -# Test framework +# Tests +/src/test/net_peer_eviction_tests.cpp @jonatack /test/functional/mempool_updatefromblock.py @hebasto /test/functional/feature_asmap.py @jonatack /test/functional/interface_bitcoin_cli.py @jonatack -/test/functional/tool_wallet.py @jonatack + +# Backwards compatibility tests +*_compatibility.py @sjors +/test/functional/wallet_upgradewallet.py @sjors @achow101 +/test/get_previous_releases.py @sjors # Translations /src/util/translation.h @hebasto @@ -98,6 +104,11 @@ # Descriptors *descriptor* @achow101 @sipa +# External signer +*external_signer* @sjors +/doc/external-signer.md @sjors +*signer.py @sjors + # Interfaces /src/interfaces/ @ryanofsky @@ -105,8 +116,7 @@ /src/txdb.* @jamesob /src/dbwrapper.* @jamesob -# Scripts/Linter -*.sh @practicalswift +# Linter /test/lint/ @practicalswift /test/lint/lint-shell.sh @hebasto diff --git a/build_msvc/README.md b/build_msvc/README.md index 5ce0f6cde4..88a05644a7 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -3,7 +3,7 @@ Building Bitcoin Core with Visual Studio Introduction --------------------- -Solution and project files to build the Bitcoin Core applications `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2017 and 2019. +Solution and project files to build the Bitcoin Core applications `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work). Building with Visual Studio is an alternative to the Linux based [cross-compiler build](https://github.com/bitcoin/bitcoin/blob/master/doc/build-windows.md). @@ -35,7 +35,7 @@ vcpkg integrate install Qt --------------------- -In order to build the Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v141, v142) and platform type (x86 or x64) must also match. +In order to build Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v142) and platform type (x86 or x64) must also match. Some prebuilt x64 versions of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware these downloads are NOT officially sanctioned by Bitcoin Core and are provided for developer convenience only. They should NOT be used for builds that will be used in a production environment or with real funds. @@ -57,19 +57,13 @@ PS >py -3 msvc-autogen.py - An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. -- To build from the command line with the Visual Studio 2017 toolchain use: - -``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v141 /t:build -``` - - To build from the command line with the Visual Studio 2019 toolchain use: ``` msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build ``` -- Alternatively open the `build_msvc/bitcoin.sln` file in Visual Studio. +- Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019. AppVeyor --------------------- @@ -81,3 +75,25 @@ For safety reasons the Bitcoin Core .appveyor.yml file has the artifact options #- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip ``` + +Security +--------------------- +[Base address randomization](https://docs.microsoft.com/en-us/cpp/build/reference/dynamicbase-use-address-space-layout-randomization?view=msvc-160) is used to make Bitcoin Core more secure. When building Bitcoin using the `build_msvc` process base address randomization can be disabled by editing `common.init.vcproj` to change `RandomizedBaseAddress` from `true` to `false` and then rebuilding the project. + +To check if `bitcoind` has `RandomizedBaseAddress` enabled or disabled run + +``` +.\dumpbin.exe /headers src/bitcoind.exe +``` + +If is it enabled then in the output `Dynamic base` will be listed in the `DLL characteristics` under `OPTIONAL HEADER VALUES` as shown below + +``` + 8160 DLL characteristics + High Entropy Virtual Addresses + Dynamic base + NX compatible + Terminal Server Aware +``` + +This may not disable all stack randomization as versions of windows employ additional stack randomization protections. These protections must be turned off in the OS configuration.
\ No newline at end of file diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index 48dfafaee0..c2c32af838 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -10,6 +10,9 @@ </PropertyGroup> <ItemGroup> <ClCompile Include="..\..\src\bitcoind.cpp" /> + <ClCompile Include="..\..\src\init\bitcoind.cpp"> + <ObjectFileName>$(IntDir)init_bitcoind.obj</ObjectFileName> + </ClCompile> </ItemGroup> <ItemGroup> <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index f195d3d451..6ea018d846 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -105,6 +105,7 @@ <Link> <SubSystem>Console</SubSystem> <AdditionalDependencies>Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> + <RandomizedBaseAddress>true</RandomizedBaseAddress> </Link> <Lib> <AdditionalOptions>/ignore:4221</AdditionalOptions> diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 490ce8b1ce..96bb584375 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -34,6 +34,7 @@ <ClCompile Include="..\..\src\qt\overviewpage.cpp" /> <ClCompile Include="..\..\src\qt\paymentserver.cpp" /> <ClCompile Include="..\..\src\qt\peertablemodel.cpp" /> + <ClCompile Include="..\..\src\qt\peertablesortproxy.cpp" /> <ClCompile Include="..\..\src\qt\platformstyle.cpp" /> <ClCompile Include="..\..\src\qt\psbtoperationsdialog.cpp" /> <ClCompile Include="..\..\src\qt\qrimagewidget.cpp" /> @@ -87,6 +88,7 @@ <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_overviewpage.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_paymentserver.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_peertablemodel.cpp" /> + <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_peertablesortproxy.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_platformstyle.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_psbtoperationsdialog.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_qrimagewidget.cpp" /> diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index d99b17d381..d99b17d381 100644..100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py diff --git a/build_msvc/testconsensus/testconsensus.cpp b/build_msvc/testconsensus/testconsensus.cpp index 115c92792d..f3c8517130 100644 --- a/build_msvc/testconsensus/testconsensus.cpp +++ b/build_msvc/testconsensus/testconsensus.cpp @@ -45,7 +45,7 @@ int main() stream << vanillaSpendTx; bitcoinconsensus_error err; - auto op0Result = bitcoinconsensus_verify_script_with_amount(pubKeyScript.data(), pubKeyScript.size(), amount, (const unsigned char*)&stream[0], stream.size(), 0, bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL, &err); + auto op0Result = bitcoinconsensus_verify_script_with_amount(pubKeyScript.data(), pubKeyScript.size(), amount, stream.data(), stream.size(), 0, bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL, &err); std::cout << "Op0 result: " << op0Result << ", error code " << err << std::endl; getchar(); diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index e6aec723bc..fa4d0410fa 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -14,6 +14,9 @@ export BASE_ROOT_DIR # The depends dir. # This folder exists on the ci host and ci guest. Changes are propagated back and forth. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} +# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) +# This folder only exists on the ci host. +export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch} echo "Setting specific values in env" if [ -n "${FILE_ENV}" ]; then @@ -25,9 +28,6 @@ fi echo "Fallback to default values in env (if not yet set)" # The number of parallel jobs to pass down to make and test_runner.py export MAKEJOBS=${MAKEJOBS:--j4} -# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) -# This folder only exists on the ci host. -export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch} # What host to compile for. See also ./depends/README.md # Tests that need cross-compilation export the appropriate HOST. # Tests that run natively guess the host diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index f78a84eeac..f78a84eeac 100644..100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 07f099b85c..07f099b85c 100644..100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 05c724fc0b..05c724fc0b 100644..100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index f051318a58..196394e908 100644..100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -10,8 +10,8 @@ export CONTAINER_NAME=ci_macos_cross export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well) export HOST=x86_64-apple-darwin18 export PACKAGES="cmake imagemagick librsvg2-bin libz-dev libtiff-tools libtinfo5 python3-setuptools xorriso" -export XCODE_VERSION=11.3.1 -export XCODE_BUILD_ID=11C505 +export XCODE_VERSION=12.1 +export XCODE_BUILD_ID=12A7403 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index e54e78add4..898c1530a1 100644..100755 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -7,12 +7,11 @@ export LC_ALL=C.UTF-8 export HOST=x86_64-apple-darwin18 -export PIP_PACKAGES="zmq" +export PIP_PACKAGES="zmq lief" export GOAL="install" export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-external-signer" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" export CCACHE_SIZE=300M - export RUN_SECURITY_TESTS="true" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 6039c51018..92af98aa9b 100644..100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_asan export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" -export DOCKER_NAME_TAG=ubuntu:20.04 +export DOCKER_NAME_TAG=ubuntu:hirsute export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++ --enable-external-signer" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index bedd0cf9aa..bedd0cf9aa 100644..100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh new file mode 100755 index 0000000000..d5b28ca5cf --- /dev/null +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export DOCKER_NAME_TAG="ubuntu:20.04" +LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" +export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" +LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" +export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" + +export CONTAINER_NAME="ci_native_msan" +export PACKAGES="clang-9 llvm-9 cmake" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export GOAL="install" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export USE_MEMORY_SANITIZER="true" +export RUN_UNIT_TESTS="false" +export RUN_FUNCTIONAL_TESTS="false" +export RUN_FUZZ_TESTS=true +export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 2cf672b91e..2cf672b91e 100644..100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 3ce50f816f..7bcf9f23a2 100644..100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export DOCKER_NAME_TAG="ubuntu:20.04" -LIBCXX_DIR="${BASE_ROOT_DIR}/ci/scratch/msan/build/" +LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" @@ -15,9 +15,9 @@ export BDB_PREFIX="${BASE_ROOT_DIR}/db4" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-9 llvm-9 cmake" -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${BASE_ROOT_DIR}/depends/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" +export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index 26a3996ce2..37d714400b 100644..100755 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -8,9 +8,10 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_multiprocess export DOCKER_NAME_TAG=ubuntu:20.04 -export PACKAGES="cmake python3" -export DEP_OPTS="MULTIPROCESS=1" +export PACKAGES="cmake python3 python3-pip llvm clang" +export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-external-signer" +export BITCOIN_CONFIG="--enable-external-signer --enable-debug CC=clang CXX=clang++" # Use clang to avoid OOM export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" export RUN_SECURITY_TESTS="true" +export PIP_PACKAGES="lief" diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index a496b5af6e..a496b5af6e 100644..100755 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 61948ab221..61948ab221 100644..100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 33f63fa9ba..33f63fa9ba 100644..100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index e079a7057c..e079a7057c 100644..100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 88b431f3c7..88b431f3c7 100644..100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 4d5bde13fd..4d5bde13fd 100644..100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 608acfc2cf..01dbfe221b 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -67,6 +67,9 @@ if [[ $DOCKER_NAME_TAG == centos* ]]; then elif [ "$CI_USE_APT_INSTALL" != "no" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get update ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES + if [ -n "$PIP_PACKAGES" ]; then + ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + fi fi if [ "$CI_OS_NAME" == "macos" ]; then @@ -92,7 +95,7 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then DOCKER_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100" DOCKER_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100" DOCKER_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" - DOCKER_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-10.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" + DOCKER_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" fi diff --git a/configure.ac b/configure.ac index d1707dfd64..2bc404250d 100644 --- a/configure.ac +++ b/configure.ac @@ -194,12 +194,6 @@ AC_ARG_ENABLE([fuzz-binary], [enable_fuzz_binary=$enableval], [enable_fuzz_binary=yes]) -AC_ARG_ENABLE([danger_fuzz_link_all], - AS_HELP_STRING([--enable-danger-fuzz-link-all], - [Danger! Modifies source code. Needs git and gnu sed installed. Link each fuzz target (default no).]), - [enable_danger_fuzz_link_all=$enableval], - [enable_danger_fuzz_link_all=no]) - AC_ARG_WITH([qrencode], [AS_HELP_STRING([--with-qrencode], [enable QR code support (default is yes if qt is enabled and libqrencode is found)])], @@ -1326,9 +1320,11 @@ fi if test x$enable_wallet != xno; then dnl Check for libdb_cxx only if wallet enabled - BITCOIN_FIND_BDB48 - if test x$suppress_external_warnings != xno ; then + if test "x$use_bdb" != "xno"; then + BITCOIN_FIND_BDB48 + if test x$suppress_external_warnings != xno ; then BDB_CPPFLAGS=SUPPRESS_WARNINGS($BDB_CPPFLAGS) + fi fi dnl Check for sqlite3 @@ -1788,7 +1784,6 @@ AM_CONDITIONAL([ENABLE_TRACING],[test x$have_sdt = xyes]) AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes]) AM_CONDITIONAL([ENABLE_FUZZ_BINARY],[test x$enable_fuzz_binary = xyes]) -AM_CONDITIONAL([ENABLE_FUZZ_LINK_ALL],[test x$enable_danger_fuzz_link_all = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 7b09c42fde..b6628c2ad5 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -6,22 +6,13 @@ Perform basic security checks on a series of executables. Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. -Needs `objdump` (for PE) and `otool` (for MACHO). ''' -import subprocess import sys -import os from typing import List, Optional +import lief import pixie -OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') -OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') - -def run_command(command) -> str: - p = subprocess.run(command, stdout=subprocess.PIPE, check=True, universal_newlines=True) - return p.stdout - def check_ELF_PIE(executable) -> bool: ''' Check for position independent executable (PIE), allowing for address space randomization. @@ -143,112 +134,59 @@ def check_ELF_separate_code(executable): return False return True -def get_PE_dll_characteristics(executable) -> int: - '''Get PE DllCharacteristics bits''' - stdout = run_command([OBJDUMP_CMD, '-x', executable]) - - bits = 0 - for line in stdout.splitlines(): - tokens = line.split() - if len(tokens)>=2 and tokens[0] == 'DllCharacteristics': - bits = int(tokens[1],16) - return bits - -IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020 -IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040 -IMAGE_DLL_CHARACTERISTICS_NX_COMPAT = 0x0100 - def check_PE_DYNAMIC_BASE(executable) -> bool: '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - bits = get_PE_dll_characteristics(executable) - return (bits & IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE) == IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE + binary = lief.parse(executable) + return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists # Must support high-entropy 64-bit address space layout randomization # in addition to DYNAMIC_BASE to have secure ASLR. def check_PE_HIGH_ENTROPY_VA(executable) -> bool: '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - bits = get_PE_dll_characteristics(executable) - return (bits & IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA) == IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA + binary = lief.parse(executable) + return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists def check_PE_RELOC_SECTION(executable) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' - stdout = run_command([OBJDUMP_CMD, '-h', executable]) - - for line in stdout.splitlines(): - if '.reloc' in line: - return True - return False - -def check_PE_NX(executable) -> bool: - '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' - bits = get_PE_dll_characteristics(executable) - return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT - -def get_MACHO_executable_flags(executable) -> List[str]: - stdout = run_command([OTOOL_CMD, '-vh', executable]) - - flags: List[str] = [] - for line in stdout.splitlines(): - tokens = line.split() - # filter first two header lines - if 'magic' in tokens or 'Mach' in tokens: - continue - # filter ncmds and sizeofcmds values - flags += [t for t in tokens if not t.isdigit()] - return flags - -def check_MACHO_PIE(executable) -> bool: - ''' - Check for position independent executable (PIE), allowing for address space randomization. - ''' - flags = get_MACHO_executable_flags(executable) - if 'PIE' in flags: - return True - return False + binary = lief.parse(executable) + return binary.has_relocations def check_MACHO_NOUNDEFS(executable) -> bool: ''' Check for no undefined references. ''' - flags = get_MACHO_executable_flags(executable) - if 'NOUNDEFS' in flags: - return True - return False - -def check_MACHO_NX(executable) -> bool: - ''' - Check for no stack execution - ''' - flags = get_MACHO_executable_flags(executable) - if 'ALLOW_STACK_EXECUTION' in flags: - return False - return True + binary = lief.parse(executable) + return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) def check_MACHO_LAZY_BINDINGS(executable) -> bool: ''' Check for no lazy bindings. We don't use or check for MH_BINDATLOAD. See #18295. ''' - stdout = run_command([OTOOL_CMD, '-l', executable]) - - for line in stdout.splitlines(): - tokens = line.split() - if 'lazy_bind_off' in tokens or 'lazy_bind_size' in tokens: - if tokens[1] != '0': - return False - return True + binary = lief.parse(executable) + return binary.dyld_info.lazy_bind == (0,0) def check_MACHO_Canary(executable) -> bool: ''' Check for use of stack canary ''' - stdout = run_command([OTOOL_CMD, '-Iv', executable]) + binary = lief.parse(executable) + return binary.has_symbol('___stack_chk_fail') - ok = False - for line in stdout.splitlines(): - if '___stack_chk_fail' in line: - ok = True - return ok +def check_PIE(executable) -> bool: + ''' + Check for position independent executable (PIE), + allowing for address space randomization. + ''' + binary = lief.parse(executable) + return binary.is_pie + +def check_NX(executable) -> bool: + ''' + Check for no stack execution + ''' + binary = lief.parse(executable) + return binary.has_nx CHECKS = { 'ELF': [ @@ -259,15 +197,16 @@ CHECKS = { ('separate_code', check_ELF_separate_code), ], 'PE': [ + ('PIE', check_PIE), ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), - ('NX', check_PE_NX), + ('NX', check_NX), ('RELOC_SECTION', check_PE_RELOC_SECTION) ], 'MACHO': [ - ('PIE', check_MACHO_PIE), + ('PIE', check_PIE), ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('NX', check_MACHO_NX), + ('NX', check_NX), ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), ('Canary', check_MACHO_Canary) ] @@ -285,24 +224,24 @@ def identify_executable(executable) -> Optional[str]: return None if __name__ == '__main__': - retval = 0 + retval: int = 0 for filename in sys.argv[1:]: try: etype = identify_executable(filename) if etype is None: - print('%s: unknown format' % filename) + print(f'{filename}: unknown format') retval = 1 continue - failed = [] + failed: List[str] = [] for (name, func) in CHECKS[etype]: if not func(filename): failed.append(name) if failed: - print('%s: failed %s' % (filename, ' '.join(failed))) + print(f'{filename}: failed {" ".join(failed)}') retval = 1 except IOError: - print('%s: cannot open' % filename) + print(f'{filename}: cannot open') retval = 1 sys.exit(retval) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 436f179d61..d740a94560 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -15,6 +15,7 @@ import sys import os from typing import List, Optional +import lief import pixie # Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases @@ -52,8 +53,6 @@ IGNORE_EXPORTS = { 'environ', '_environ', '__environ', } CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') -OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') -OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { @@ -203,44 +202,22 @@ def check_ELF_libraries(filename) -> bool: ok = False return ok -def macho_read_libraries(filename) -> List[str]: - p = subprocess.Popen([OTOOL_CMD, '-L', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - libraries = [] - for line in stdout.splitlines(): - tokens = line.split() - if len(tokens) == 1: # skip executable name - continue - libraries.append(tokens[0].split('/')[-1]) - return libraries - def check_MACHO_libraries(filename) -> bool: ok: bool = True - for dylib in macho_read_libraries(filename): - if dylib not in MACHO_ALLOWED_LIBRARIES: - print('{} is not in ALLOWED_LIBRARIES!'.format(dylib)) + binary = lief.parse(filename) + for dylib in binary.libraries: + split = dylib.name.split('/') + if split[-1] not in MACHO_ALLOWED_LIBRARIES: + print(f'{split[-1]} is not in ALLOWED_LIBRARIES!') ok = False return ok -def pe_read_libraries(filename) -> List[str]: - p = subprocess.Popen([OBJDUMP_CMD, '-x', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - libraries = [] - for line in stdout.splitlines(): - if 'DLL Name:' in line: - tokens = line.split(': ') - libraries.append(tokens[1]) - return libraries - def check_PE_libraries(filename) -> bool: ok: bool = True - for dylib in pe_read_libraries(filename): + binary = lief.parse(filename) + for dylib in binary.libraries: if dylib not in PE_ALLOWED_LIBRARIES: - print('{} is not in ALLOWED_LIBRARIES!'.format(dylib)) + print(f'{dylib} is not in ALLOWED_LIBRARIES!') ok = False return ok @@ -275,7 +252,7 @@ if __name__ == '__main__': try: etype = identify_executable(filename) if etype is None: - print('{}: unknown format'.format(filename)) + print(f'{filename}: unknown format') retval = 1 continue @@ -284,9 +261,9 @@ if __name__ == '__main__': if not func(filename): failed.append(name) if failed: - print('{}: failed {}'.format(filename, ' '.join(failed))) + print(f'{filename}: failed {" ".join(failed)}') retval = 1 except IOError: - print('{}: cannot open'.format(filename)) + print(f'{filename}: cannot open') retval = 1 sys.exit(retval) diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 60acb0d593..5df87d9e70 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -210,7 +210,7 @@ def main(): args.macos = 'm' in args.os # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'): + if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'): print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') args.macos = False diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME index 330fbc041b..330fbc041b 100755..100644 --- a/contrib/gitian-descriptors/assign_DISTNAME +++ b/contrib/gitian-descriptors/assign_DISTNAME diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 9b5f5d7e07..103e249e33 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -23,6 +23,7 @@ packages: - "patch" - "pkg-config" - "python3" +- "python3-pip" # Cross compilation HOSTS: # - arm-linux-gnueabihf - "binutils-arm-linux-gnueabihf" @@ -99,6 +100,8 @@ script: | done } + pip3 install lief==0.11.4 + # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index c1bd545501..d6c41b2c43 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -23,6 +23,7 @@ packages: - "imagemagick" - "libz-dev" - "python3" +- "python3-pip" - "python3-setuptools" - "fonts-tuffy" - "xorriso" @@ -31,7 +32,7 @@ remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: -- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz" +- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz" script: | set -e -o pipefail @@ -78,6 +79,8 @@ script: | done } + pip3 install lief==0.11.4 + # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" @@ -88,7 +91,7 @@ script: | BASEPREFIX="${PWD}/depends" mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz # Build dependencies for each host for i in $HOSTS; do diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index dbf1e8cc67..eabcdaa79d 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -22,6 +22,7 @@ packages: - "zip" - "ca-certificates" - "python3" +- "python3-pip" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -86,6 +87,8 @@ script: | done } + pip3 install lief==0.11.4 + # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" diff --git a/contrib/guix/README.md b/contrib/guix/README.md index dad7de32c4..e604b370e3 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -167,7 +167,7 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum Set the path where _extracted_ SDKs can be found. This is passed through to the depends tree. Note that this is should be set to the _parent_ directory of the actual SDK (e.g. SDK_PATH=$HOME/Downloads/macOS-SDKs instead of - $HOME/Downloads/macOS-SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers). + $HOME/Downloads/macOS-SDKs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers). * _**JOBS**_ diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 6ea32329ba..e95617e2e2 100644..100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -255,7 +255,7 @@ case "$HOST" in esac case "$HOST" in - powerpc64-linux-*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,-z,noexecstack" ;; + powerpc64-linux-*|riscv64-linux-*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,-z,noexecstack" ;; esac # Make $HOST-specific native binaries from depends available in $PATH diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 910a9dd6f6..f98f2b9422 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -27,9 +27,11 @@ (gnu packages version-control) (guix build-system font) (guix build-system gnu) + (guix build-system python) (guix build-system trivial) (guix download) (guix gexp) + (guix git-download) ((guix licenses) #:prefix license:) (guix packages) (guix profiles) @@ -129,7 +131,7 @@ chain for " target " development.")) (base-gcc-for-libc gcc-7) (base-kernel-headers linux-libre-headers-5.4) (base-libc glibc) ; glibc 2.31 - (base-gcc (make-gcc-rpath-link gcc-9))) + (base-gcc (make-gcc-rpath-link gcc-8))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target @@ -147,7 +149,7 @@ desirable for building Bitcoin Core release binaries." (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target - #:xgcc (make-ssp-fixed-gcc gcc-9) + #:xgcc (make-ssp-fixed-gcc gcc-8) #:xbinutils xbinutils #:libc pthreads-xlibc)))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and @@ -192,6 +194,29 @@ chain for " target " development.")) "Thatcher Ulrich's first outline font design. He started with the goal of producing a neutral, readable sans-serif text font. There are lots of \"expressive\" fonts out there, but he wanted to start with something very plain and clean, something he might want to actually use. ") (license license:public-domain))) +(define-public lief + (package + (name "python-lief") + (version "0.11.4") + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/lief-project/LIEF.git") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0h4kcwr9z478almjqhmils8imfpflzk0r7d05g4xbkdyknn162qf")))) + (build-system python-build-system) + (native-inputs + `(("cmake" ,cmake))) + (home-page "https://github.com/lief-project/LIEF") + (synopsis "Library to Instrument Executable Formats") + (description "Python library to to provide a cross platform library which can +parse, modify and abstract ELF, PE and MachO formats.") + (license license:asl2.0))) + (packages->manifest (append (list ;; The Basics @@ -227,6 +252,8 @@ chain for " target " development.")) python-3 ;; Git git + ;; Tests + lief ;; Native gcc 7 toolchain gcc-toolchain-7 (list gcc-toolchain-7 "static")) @@ -239,5 +266,5 @@ chain for " target " development.")) ((string-contains target "-linux-") (list (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-toolchain-8 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso)) + (list clang-toolchain-10 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso)) (else '()))))) diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 2d9a4a2153..21f6ba2eb3 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -13,9 +13,9 @@ When complete, it will have produced `Bitcoin-Core.dmg`. ### Step 1: Obtaining `Xcode.app` Our current macOS SDK -(`Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`) can be +(`Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`) can be extracted from -[Xcode_11.3.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). +[Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip). An Apple ID is needed to download this. After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` @@ -27,25 +27,25 @@ approach (tested on Debian Buster) is outlined below: apt install cpio git clone https://github.com/bitcoin-core/apple-sdk-tools.git -# Unpack Xcode_11.3.1.xip and place the resulting Xcode.app in your current +# Unpack Xcode_12.1.xip and place the resulting Xcode.app in your current # working directory -python3 apple-sdk-tools/extract_xcode.py -f Xcode_11.3.1.xip | cpio -d -i +python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.1.xip | cpio -d -i ``` On macOS the process is more straightforward: ```bash -xip -x Xcode_11.3.1.xip +xip -x Xcode_12.1.xip ``` -### Step 2: Generating `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` +### Step 2: Generating `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` -To generate `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`, run +To generate `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`, run the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the previous stage) as the first argument. ```bash -# Generate a Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz from +# Generate a Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz from # the supplied Xcode.app ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` @@ -56,7 +56,7 @@ Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple Apple uses `clang` extensively for development and has upstreamed the necessary functionality so that a vanilla clang can take advantage. It supports the use of `-F`, -`-target`, `-mmacosx-version-min`, and `--sysroot`, which are all necessary when +`-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when building for macOS. Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the diff --git a/contrib/qos/tc.sh b/contrib/qos/tc.sh index 1cde19efd1..1cde19efd1 100644..100755 --- a/contrib/qos/tc.sh +++ b/contrib/qos/tc.sh diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index d95069277d..9560b586ec 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -16,6 +16,7 @@ These files must consist of lines in the format <ip>:<port> [<ipv6>]:<port> <onion>.onion:<port> + <i2p>.b32.i2p:<port> The output will be two data structures with the peers in binary format: @@ -52,6 +53,12 @@ def name_to_bip155(addr): return (BIP155Network.TORV3, vchAddr[:32]) else: raise ValueError('Invalid onion %s' % vchAddr) + elif addr.endswith('.b32.i2p'): + vchAddr = b32decode(addr[0:-8] + '====', True) + if len(vchAddr) == 32: + return (BIP155Network.I2P, vchAddr) + else: + raise ValueError(f'Invalid I2P {vchAddr}') elif '.' in addr: # IPv4 return (BIP155Network.IPV4, bytes((int(x) for x in addr.split('.')))) elif ':' in addr: # IPv6 diff --git a/contrib/seeds/nodes_main.txt b/contrib/seeds/nodes_main.txt index c43caa73eb..7300e3043d 100644 --- a/contrib/seeds/nodes_main.txt +++ b/contrib/seeds/nodes_main.txt @@ -1188,3 +1188,13 @@ sys54sv4xv3hn3sdiv3oadmzqpgyhd4u4xphv4xqk64ckvaxzm57a7yd.onion:8333 tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333 vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333 xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333 + +# manually added 2021-05 for minimal i2p bootstrap support +72l3ucjkuscrbiiepoehuwqgknyzgo7zuix5ty4puwrkyhtmnsga.b32.i2p:8333 +c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:8333 +gehtac45oaghz54ypyopim64mql7oad2bqclla74l6tfeolzmodq.b32.i2p:8333 +h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:8333 +hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:8333 +pjs7or2ctvteeo5tu4bwyrtydeuhqhvdprtujn4daxr75jpebjxa.b32.i2p:8333 +wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:8333 +zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:8333 diff --git a/depends/Makefile b/depends/Makefile index c5dfd1e8a7..ac12e91e49 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -34,6 +34,8 @@ BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs NO_QT ?= NO_QR ?= +NO_BDB ?= +NO_SQLITE ?= NO_WALLET ?= NO_ZMQ ?= NO_UPNP ?= @@ -112,37 +114,31 @@ include builders/$(build_os).mk include builders/default.mk include packages/packages.mk -full_env=$(shell printenv) - -build_id_string:=$(BUILD_ID_SALT) - -# GCC only prints COLLECT_LTO_WRAPPER when invoked with just "-v", but we want -# the information from "-v -E -" as well, so just include both. +# Previously, we directly invoked the well-known programs using $(shell ...) +# to contruct build_id_string. However, that was problematic because: # -# '3>&1 1>&2 2>&3 > /dev/null' is supposed to swap stdin and stdout and silence -# stdin, since we only want the stderr output -build_id_string+=$(shell $(build_CC) -v < /dev/null 3>&1 1>&2 2>&3 > /dev/null) $(shell $(build_CC) -v -E - < /dev/null 3>&1 1>&2 2>&3 > /dev/null) -build_id_string+=$(shell $(build_AR) --version 2>/dev/null) $(filter AR_%,$(full_env)) ZERO_AR_DATE=$(ZERO_AR_DATE) -build_id_string+=$(shell $(build_CXX) -v < /dev/null 3>&1 1>&2 2>&3 > /dev/null) $(shell $(build_CXX) -v -E - < /dev/null 3>&1 1>&2 2>&3 > /dev/null) -build_id_string+=$(shell $(build_RANLIB) --version 2>/dev/null) $(filter RANLIB_%,$(full_env)) -build_id_string+=$(shell $(build_STRIP) --version 2>/dev/null) $(filter STRIP_%,$(full_env)) - -$(host_arch)_$(host_os)_id_string:=$(HOST_ID_SALT) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_CC) -v < /dev/null 3>&1 1>&2 2>&3 > /dev/null) $(shell $(host_CC) -v -E - < /dev/null 3>&1 1>&2 2>&3 > /dev/null) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_AR) --version 2>/dev/null) $(filter AR_%,$(full_env)) ZERO_AR_DATE=$(ZERO_AR_DATE) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) -v < /dev/null 3>&1 1>&2 2>&3 > /dev/null) $(shell $(host_CXX) -v -E - < /dev/null 3>&1 1>&2 2>&3 > /dev/null) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(filter RANLIB_%,$(full_env)) -$(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) $(filter STRIP_%,$(full_env)) - -ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -# Make sure that cache is invalidated when switching between system and -# depends-managed, pinned clang -build_id_string+=system_clang -$(host_arch)_$(host_os)_id_string+=system_clang -endif - -build_id_string+=GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT)) -$(host_arch)_$(host_os)_id_string+=GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT)) +# 1. When invoking a shell, GNU Make special-cases exit code 127 (command not +# found) by not capturing the output but instead passing it through. This is +# not done for any other exit code. +# +# 2. Characters like '#' (from these programs' output) would end up in make +# variables like build_id_string, which would be wrongly interpreted by make +# when these variables were used. +# +# Therefore, we should avoid having arbitrary strings in make variables where +# possible. The gen_id script used here hashes the output to construct a +# "make-safe" id. +# +# Also note that these lines need to be: +# +# 1. After including {hosts,builders}/*.mk, since they rely on the tool +# variables (e.g. build_CC, host_STRIP, etc.) to be set. +# +# 2. Before including packages/*.mk (excluding packages/packages.mk), since +# they rely on the build_id variables +# +build_id:=$(shell env CC='$(build_CC)' CXX='$(build_CXX)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' CXX='$(host_CXX)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') qrencode_packages_$(NO_QR) = $(qrencode_packages) @@ -233,6 +229,8 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@no_qr@|$(NO_QR)|' \ -e 's|@no_zmq@|$(NO_ZMQ)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ + -e 's|@no_bdb@|$(NO_BDB)|' \ + -e 's|@no_sqlite@|$(NO_SQLITE)|' \ -e 's|@no_upnp@|$(NO_UPNP)|' \ -e 's|@no_natpmp@|$(NO_NATPMP)|' \ -e 's|@multiprocess@|$(MULTIPROCESS)|' \ diff --git a/depends/README.md b/depends/README.md index 6b20791281..50e1a32c70 100644 --- a/depends/README.md +++ b/depends/README.md @@ -49,6 +49,11 @@ The paths are automatically configured and no other options are needed unless ta sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libz-dev python3-setuptools libtinfo5 xorriso +Note: You must obtain the macOS SDK before proceeding with a cross-compile. +Under the depends directory, create a subdirectory named `SDKs`. +Then, place the extracted SDK under this new directory. +For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-extraction). + #### For Win64 cross compilation - see [build-windows.md](../doc/build-windows.md#cross-compilation-for-ubuntu-and-windows-subsystem-for-linux) diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index f4103fc1f2..001c928424 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -1,5 +1,5 @@ -build_darwin_CC:=$(shell xcrun -f clang) --sysroot $(shell xcrun --show-sdk-path) -build_darwin_CXX:=$(shell xcrun -f clang++) --sysroot $(shell xcrun --show-sdk-path) +build_darwin_CC:=$(shell xcrun -f clang) -isysroot$(shell xcrun --show-sdk-path) +build_darwin_CXX:=$(shell xcrun -f clang++) -isysroot$(shell xcrun --show-sdk-path) build_darwin_AR:=$(shell xcrun -f ar) build_darwin_RANLIB:=$(shell xcrun -f ranlib) build_darwin_STRIP:=$(shell xcrun -f strip) @@ -10,8 +10,8 @@ build_darwin_SHA256SUM=shasum -a 256 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o #darwin host on darwin builder. overrides darwin host preferences. -darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(shell xcrun --show-sdk-path) -darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ --sysroot $(shell xcrun --show-sdk-path) +darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroot$(shell xcrun --show-sdk-path) +darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ -isysroot$(shell xcrun --show-sdk-path) darwin_AR:=$(shell xcrun -f ar) darwin_RANLIB:=$(shell xcrun -f ranlib) darwin_STRIP:=$(shell xcrun -f strip) diff --git a/depends/config.site.in b/depends/config.site.in index 97c264ff0b..5cf107f19b 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -38,6 +38,14 @@ if test -z "$enable_wallet" && test -n "@no_wallet@"; then enable_wallet=no fi +if test -z "$with_bdb" && test -n "@no_bdb@"; then + with_bdb=no +fi + +if test -z "$with_sqlite" && test -n "@no_sqlite@"; then + with_sqlite=no +fi + if test -z "$enable_multiprocess" && test -n "@multiprocess@"; then enable_multiprocess=yes fi diff --git a/depends/funcs.mk b/depends/funcs.mk index 6e1a922f89..34a030fab7 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -49,7 +49,7 @@ define int_get_build_id $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies)) $(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies))) $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) -$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string)) +$(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id)) $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) final_build_id_long+=$($(package)_build_id_long) diff --git a/depends/gen_id b/depends/gen_id new file mode 100755 index 0000000000..ac69ca7ee1 --- /dev/null +++ b/depends/gen_id @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Usage: env [ CC=... ] [ CXX=... ] [ AR=... ] [ RANLIB=... ] [ STRIP=... ] \ +# [ DEBUG=... ] ./build-id [ID_SALT]... +# +# Prints to stdout a SHA256 hash representing the current toolset, used by +# depends/Makefile as a build id for caching purposes (detecting when the +# toolset has changed and the cache needs to be invalidated). +# +# If the DEBUG environment variable is non-empty and the system has `tee` +# available in its $PATH, the pre-image to the SHA256 hash will be printed to +# stderr. This is to help developers debug caching issues in depends. + +# This script explicitly does not `set -e` because id determination is mostly +# opportunistic: it is fine that things fail, as long as they fail consistently. + +# Command variables (CC/CXX/AR) which can be blank are invoked with `bash -c`, +# because the "command not found" error message printed by shells often include +# the line number, like so: +# +# ./depends/gen_id: line 43: --version: command not found +# +# By invoking with `bash -c`, we ensure that the line number is always 1 + +( + # Redirect stderr to stdout + exec 2>&1 + + echo "BEGIN ALL" + + # Include any ID salts supplied via command line + echo "BEGIN ID SALT" + echo "$@" + echo "END ID SALT" + + # GCC only prints COLLECT_LTO_WRAPPER when invoked with just "-v", but we want + # the information from "-v -E -" as well, so just include both. + echo "BEGIN CC" + bash -c "${CC} -v" + bash -c "${CC} -v -E -xc -o /dev/null - < /dev/null" + bash -c "${CC} -v -E -xobjective-c -o /dev/null - < /dev/null" + echo "END CC" + + echo "BEGIN CXX" + bash -c "${CXX} -v" + bash -c "${CXX} -v -E -xc++ -o /dev/null - < /dev/null" + bash -c "${CXX} -v -E -xobjective-c++ -o /dev/null - < /dev/null" + echo "END CXX" + + echo "BEGIN AR" + bash -c "${AR} --version" + env | grep '^AR_' + echo "ZERO_AR_DATE=${ZERO_AR_DATE}" + echo "END AR" + + echo "BEGIN RANLIB" + bash -c "${RANLIB} --version" + env | grep '^RANLIB_' + echo "END RANLIB" + + echo "BEGIN STRIP" + bash -c "${STRIP} --version" + env | grep '^STRIP_' + echo "END STRIP" + + echo "END ALL" +) | if [ -n "$DEBUG" ] && command -v tee > /dev/null 2>&1; then + # When debugging and `tee` is available, output the preimage to stderr + # in addition to passing through stdin to stdout + tee >(cat 1>&2) + else + # Otherwise, passthrough stdin to stdout + cat + fi | ${SHA256SUM} - | cut -d' ' -f1 diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index dd71697f0f..5a7ae2df9a 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,8 +1,8 @@ OSX_MIN_VERSION=10.14 -OSX_SDK_VERSION=10.15.1 -XCODE_VERSION=11.3.1 -XCODE_BUILD_ID=11C505 -LD64_VERSION=530 +OSX_SDK_VERSION=10.15.6 +XCODE_VERSION=12.1 +XCODE_BUILD_ID=12A7403 +LD64_VERSION=609 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers @@ -60,16 +60,11 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$( # Explicitly point to our binaries (e.g. cctools) so that they are # ensured to be found and preferred over other possibilities. # -# -stdlib=libc++ -nostdinc++ -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 +# -stdlib=libc++ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 # # Forces clang to use the libc++ headers from our SDK and completely # forget about the libc++ headers from the standard directories # -# TODO: Once we start requiring a clang version that has the -# -stdlib++-isystem<directory> flag first introduced here: -# https://reviews.llvm.org/D64089, we should use that instead. Read the -# differential summary there for more details. -# # -Xclang -*system<path_a> \ # -Xclang -*system<path_b> \ # -Xclang -*system<path_c> ... @@ -100,7 +95,7 @@ darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u LIBRARY_PATH \ $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ - --sysroot=$(OSX_SDK) \ + -isysroot$(OSX_SDK) \ -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ @@ -108,9 +103,9 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u LIBRARY_PATH \ $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ - --sysroot=$(OSX_SDK) \ - -stdlib=libc++ -nostdinc++ \ - -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \ + -isysroot$(OSX_SDK) \ + -stdlib=libc++ \ + -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \ -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index 8ab448ce5f..07da752492 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -7,7 +7,7 @@ linux_release_CXXFLAGS=$(linux_release_CFLAGS) linux_debug_CFLAGS=-O1 linux_debug_CXXFLAGS=$(linux_debug_CFLAGS) -linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC +linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -D_LIBCPP_DEBUG=1 ifeq (86,$(findstring 86,$(build_arch))) i686_linux_CC=gcc -m32 diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 29a3efdfe6..0800c63dfc 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,6 +1,6 @@ package=boost $(package)_version=1_71_0 -$(package)_download_path=https://dl.bintray.com/boostorg/release/$(subst _,.,$($(package)_version))/source/ +$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$(subst _,.,$($(package)_version))/source/ $(package)_file_name=boost_$($(package)_version).tar.bz2 $(package)_sha256_hash=d73a8da01e8bf8c7eda40b4c84915071a8c8a0df4a6734537ddde4a8580524ee $(package)_dependencies=native_b2 diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index c789dab74c..885207fce9 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -1,10 +1,9 @@ package=native_cctools -$(package)_version=55562e4073dea0fbfd0b20e0bf69ffe6390c7f97 +$(package)_version=2ef2e931cf641547eb8a68cfebde61003587c9fd $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=e51995a843533a3dac155dd0c71362dd471597a2d23f13dff194c6285362f875 +$(package)_sha256_hash=6b73269efdf5c58a070e7357b66ee760501388549d6a12b423723f45888b074b $(package)_build_subdir=cctools -$(package)_patches=ld64_disable_threading.patch $(package)_dependencies=native_libtapi define $(package)_set_vars @@ -17,10 +16,6 @@ define $(package)_set_vars $(package)_cxx=$(clangxx_prog) endef -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/ld64_disable_threading.patch -endef - define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk index 741ddacf09..36adeb196d 100644 --- a/depends/packages/native_clang.mk +++ b/depends/packages/native_clang.mk @@ -1,9 +1,9 @@ package=native_clang -$(package)_version=8.0.0 -$(package)_download_path=https://releases.llvm.org/$($(package)_version) -$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_file_name=clang-llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0 +$(package)_version=10.0.1 +$(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version) +$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz +$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz +$(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231 define $(package)_preprocess_cmds rm -f $($(package)_extract_dir)/lib/libc++abi.so* diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index c50fdc3f6b..14653ce9fb 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,8 @@ package=native_libmultiprocess -$(package)_version=5741d750a04e644a03336090d8979c6d033e32c0 +$(package)_version=d576d975debdc9090bd2582f83f49c76c0061698 $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=ac848db49a6ed53e423c62d54bd87f1f08cbb0326254a8667e10bbfe5bf032a4 +$(package)_sha256_hash=9f8b055c8bba755dc32fe799b67c20b91e7b13e67cadafbc54c0f1def057a370 $(package)_dependencies=native_capnp define $(package)_config_cmds diff --git a/depends/packages/native_libtapi.mk b/depends/packages/native_libtapi.mk index d7ac4156a7..60b898da5f 100644 --- a/depends/packages/native_libtapi.mk +++ b/depends/packages/native_libtapi.mk @@ -1,9 +1,9 @@ package=native_libtapi -$(package)_version=3efb201881e7a76a21e0554906cf306432539cef +$(package)_version=664b8414f89612f2dfd35a9b679c345aa5389026 $(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive $(package)_download_file=$($(package)_version).tar.gz $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3 +$(package)_sha256_hash=62e419c12d1c9fad67cc1cd523132bc00db050998337c734c15bc8d73cc02b61 ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) $(package)_dependencies=native_clang diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index 5fe027fb8a..783f87ca7c 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -1,8 +1,8 @@ package=native_mac_alias -$(package)_version=2.1.1 +$(package)_version=2.2.0 $(package)_download_path=https://github.com/al45tair/mac_alias/archive/ $(package)_file_name=v$($(package)_version).tar.gz -$(package)_sha256_hash=c0ffceee14f7d04a6eb323fb7b8217dc3f373b346198d2ca42300a8362db7efa +$(package)_sha256_hash=421e6d7586d1f155c7db3e7da01ca0dacc9649a509a253ad7077b70174426499 $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 0133eee920..8b5bfa6895 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -9,7 +9,7 @@ $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch $(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch $(package)_patches+= drop_lrelease_dependency.patch no_sdk_version_check.patch -$(package)_patches+= fix_qpainter_non_determinism.patch fix_lib_paths.patch fix_android_pch.patch +$(package)_patches+= fix_lib_paths.patch fix_android_pch.patch $(package)_patches+= fix_bigsur_drawing.patch qtbase-moc-ignore-gcc-macro.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) @@ -227,7 +227,6 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_android_pch.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_qpainter_non_determinism.patch &&\ patch -p1 -i $($(package)_patch_dir)/no_sdk_version_check.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_lib_paths.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_bigsur_drawing.patch && \ diff --git a/depends/patches/native_cctools/ld64_disable_threading.patch b/depends/patches/native_cctools/ld64_disable_threading.patch deleted file mode 100644 index 2de6874cd4..0000000000 --- a/depends/patches/native_cctools/ld64_disable_threading.patch +++ /dev/null @@ -1,26 +0,0 @@ -commit 584668415039adeed073decee7e04de28248afd3 -Author: fanquake <fanquake@gmail.com> -Date: Tue Aug 18 01:20:24 2020 +0000 - - Disable threading to fix non-determinism - - A bug in the file parser can cause dependencies to be calculated - differently based on which files have already been parsed. This is more - likely to occur on systems with more CPUs. - - Just disable threading for now. There is no noticeable slowdown. - - See #9891. - -diff --git a/cctools/ld64/src/ld/InputFiles.h b/cctools/ld64/src/ld/InputFiles.h -index ef9c756..90a70b6 100644 ---- a/cctools/ld64/src/ld/InputFiles.h -+++ b/cctools/ld64/src/ld/InputFiles.h -@@ -25,7 +25,6 @@ - #ifndef __INPUT_FILES_H__ - #define __INPUT_FILES_H__ - --#define HAVE_PTHREADS 1 - - #include <stdlib.h> - #include <sys/types.h> diff --git a/depends/patches/qt/fix_qpainter_non_determinism.patch b/depends/patches/qt/fix_qpainter_non_determinism.patch deleted file mode 100644 index 44c45187c5..0000000000 --- a/depends/patches/qt/fix_qpainter_non_determinism.patch +++ /dev/null @@ -1,63 +0,0 @@ -commit 2a8f7dc6ddfc414a66491522501c1574a1343ee1 -Author: Andrew Chow <achow101-github@achow101.com> -Date: Sat Nov 21 01:11:04 2020 -0500 - - build: Fix determinism issue when building with Clang 8 - - When building Qt with LLVM/Clang 8 under -O3 (the default), we run into - a determinism issue in `qt_interset_spans`. The issue has been fixed for - LLVM/Clang 9, see - https://github.com/llvm/llvm-project/commit/db101864bdc938deb1d63fe4f7da761bd38e5cae - and https://reviews.llvm.org/D64601, however this fix was not backported - to 8.x. Once LLVM/Clang 9 is used, this patch can be dropped. - - The particular issue appears to be an optimization done by -O3 which - adds a temporary variable for `spans->y` in `qt_intersect_spans`. When - it does this, sometimes it chooses to use a 32-bit movs instruction - (movswl), and other times it chooses a 64-bit movs instruction (movswq). - By patching `qt_intersect_spans` to always make a temporary variable for - `spans->y`, we are able to sidestep this problem. - -diff --git a/qtbase/src/gui/painting/qpaintengine_raster.cpp b/qtbase/src/gui/painting/qpaintengine_raster.cpp -index 92ab6e8375..f018009e0b 100644 ---- a/qtbase/src/gui/painting/qpaintengine_raster.cpp -+++ b/qtbase/src/gui/painting/qpaintengine_raster.cpp -@@ -4128,22 +4128,23 @@ static const QSpan *qt_intersect_spans(const QClipData *clip, int *currentClip, - const QSpan *clipEnd = clip->m_spans + clip->count; - - while (available && spans < end ) { -+ const short spans_y = spans->y; - if (clipSpans >= clipEnd) { - spans = end; - break; - } -- if (clipSpans->y > spans->y) { -+ if (clipSpans->y > spans_y) { - ++spans; - continue; - } -- if (spans->y != clipSpans->y) { -- if (spans->y < clip->count && clip->m_clipLines[spans->y].spans) -- clipSpans = clip->m_clipLines[spans->y].spans; -+ if (spans_y != clipSpans->y) { -+ if (spans_y < clip->count && clip->m_clipLines[spans_y].spans) -+ clipSpans = clip->m_clipLines[spans_y].spans; - else - ++clipSpans; - continue; - } -- Q_ASSERT(spans->y == clipSpans->y); -+ Q_ASSERT(spans_y == clipSpans->y); - - int sx1 = spans->x; - int sx2 = sx1 + spans->len; -@@ -4162,7 +4163,7 @@ static const QSpan *qt_intersect_spans(const QClipData *clip, int *currentClip, - if (len) { - out->x = qMax(sx1, cx1); - out->len = qMin(sx2, cx2) - out->x; -- out->y = spans->y; -+ out->y = spans_y; - out->coverage = qt_div_255(spans->coverage * clipSpans->coverage); - ++out; - --available; - diff --git a/doc/files.md b/doc/files.md index a80adf30ea..695ea6f448 100644 --- a/doc/files.md +++ b/doc/files.md @@ -53,6 +53,7 @@ Subdirectory | File(s) | Description `indexes/txindex/` | LevelDB database | Transaction index; *optional*, used if `-txindex=1` `indexes/blockfilter/basic/db/` | LevelDB database | Blockfilter index LevelDB database for the basic filtertype; *optional*, used if `-blockfilterindex=basic` `indexes/blockfilter/basic/` | `fltrNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Blockfilter index filters for the basic filtertype; *optional*, used if `-blockfilterindex=basic` +`indexes/coinstats/db/` | LevelDB database | Coinstats index; *optional*, used if `-coinstatsindex=1` `wallets/` | | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, wallets reside in the [data directory](#data-directory-location) `./` | `anchors.dat` | Anchor IP address database, created on shutdown and deleted at startup. Anchors are last known outgoing block-relay-only peers that are tried to re-connect to on startup `./` | `banlist.dat` | Stores the IPs/subnets of banned nodes diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 4d8825f4c2..5ffa3a9416 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -230,3 +230,15 @@ $ honggfuzz/honggfuzz --exit_upon_crash --quiet --timeout 4 -n 1 -Q \ -nodebuglogfile -bind=127.0.0.1:18444 -logthreadnames \ -debug ``` + +# OSS-Fuzz + +Bitcoin Core participates in Google's [OSS-Fuzz](https://github.com/google/oss-fuzz/tree/master/projects/bitcoin-core) +program, which includes a dashboard of [publicly disclosed vulnerabilities](https://bugs.chromium.org/p/oss-fuzz/issues/list). +Generally, we try to disclose vulnerabilities as soon as possible after they +are fixed to give users the knowledge they need to be protected. However, +because Bitcoin is a live P2P network, and not just standalone local software, +we might not fully disclose every issue within Google's standard +[90-day disclosure window](https://google.github.io/oss-fuzz/getting-started/bug-disclosure-guidelines/) +if a partial or delayed disclosure is important to protect users or the +function of the network. diff --git a/doc/multiprocess.md b/doc/multiprocess.md index 7a42fdd734..e3f389a6d3 100644 --- a/doc/multiprocess.md +++ b/doc/multiprocess.md @@ -15,7 +15,7 @@ Specific next steps after [#10102](https://github.com/bitcoin/bitcoin/pull/10102 ## Debugging -After [#10102](https://github.com/bitcoin/bitcoin/pull/10102), the `-debug=ipc` command line option can be used to see requests and responses between processes. +The `-debug=ipc` command line option can be used to see requests and responses between processes. ## Installation @@ -33,3 +33,40 @@ BITCOIND=bitcoin-node test/functional/test_runner.py The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. + +## IPC implementation details + +Cross process Node, Wallet, and Chain interfaces are defined in +[`src/interfaces/`](../src/interfaces/). These are C++ classes which follow +[conventions](developer-notes.md#internal-interface-guidelines), like passing +serializable arguments so they can be called from different processes, and +making methods pure virtual so they can have proxy implementations that forward +calls between processes. + +When Wallet, Node, and Chain code is running in the same process, calling any +interface method invokes the implementation directly. When code is running in +different processes, calling an interface method invokes a proxy interface +implementation that communicates with a remote process and invokes the real +implementation in the remote process. The +[libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) code +generation tool internally generates proxy client classes and proxy server +classes for this purpose that are thin wrappers around Cap'n Proto +[client](https://capnproto.org/cxxrpc.html#clients) and +[server](https://capnproto.org/cxxrpc.html#servers) classes, which handle the +actual serialization and socket communication. + +As much as possible, calls between processes are meant to work the same as +calls within a single process without adding limitations or requiring extra +implementation effort. Processes communicate with each other by calling regular +[C++ interface methods](../src/interfaces/README.md). Method arguments and +return values are automatically serialized and sent between processes. Object +references and `std::function` arguments are automatically tracked and mapped +to allow invoked code to call back into invoking code at any time, and there is +a 1:1 threading model where any thread invoking a method in another process has +a corresponding thread in the invoked process responsible for executing all +method calls from the source thread, without blocking I/O or holding up another +call, and using the same thread local variables, locks, and callbacks between +calls. The forwarding, tracking, and threading is implemented inside the +[libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) library +which has the design goal of making calls between processes look like calls in +the same process to the extent possible. diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index 6e7a578ecc..296b172bde 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -24,9 +24,13 @@ The size of some in-memory caches can be reduced. As caches trade off memory usa ## Number of peers -- `-maxconnections=<n>` - the maximum number of connections, this defaults to 125. Each active connection takes up some - memory. This option applies only if incoming connections are enabled, otherwise the number of connections will never - be more than 10. Of the 10 outbound peers, there can be 8 full-relay connections and 2 block-relay-only ones. +- `-maxconnections=<n>` - the maximum number of connections, which defaults to 125. Each active connection takes up some + memory. This option applies only if inbound connections are enabled; otherwise, the number of connections will not + be more than 11. Of the 11 outbound peers, there can be 8 full-relay connections, 2 block-relay-only ones, + and occasionally 1 short-lived feeler or extra outbound block-relay-only connection. + +- These limits do not apply to connections added manually with the `-addnode` configuration option or + the `addnode` RPC, which have a separate limit of 8 connections. ## Thread configuration diff --git a/doc/release-notes-20867.md b/doc/release-notes-20867.md new file mode 100644 index 0000000000..60eed6838f --- /dev/null +++ b/doc/release-notes-20867.md @@ -0,0 +1,11 @@ +Wallet +------ + +- We now support up to 20 keys in `multi()` and `sortedmulti()` descriptors + under `wsh()`. (#20867) + +Updated RPCs +------------ + +- `addmultisigaddress` and `createmultisig` now support up to 20 keys for + Segwit addresses. diff --git a/doc/release-notes/release-notes-0.21.1.md b/doc/release-notes/release-notes-0.21.1.md new file mode 100644 index 0000000000..d032fa8429 --- /dev/null +++ b/doc/release-notes/release-notes-0.21.1.md @@ -0,0 +1,203 @@ +0.21.1 Release Notes +==================== + +Bitcoin Core version 0.21.1 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.21.1/> + +This minor release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + +Notable changes +=============== + +## Taproot Soft Fork + +Included in this release are the mainnet and testnet activation +parameters for the taproot soft fork (BIP341) which also adds support +for schnorr signatures (BIP340) and tapscript (BIP342). + +If activated, these improvements will allow users of single-signature +scripts, multisignature scripts, and complex contracts to all use +identical-appearing commitments that enhance their privacy and the +fungibility of all bitcoins. Spenders will enjoy lower fees and the +ability to resolve many multisig scripts and complex contracts with the +same efficiency, low fees, and large anonymity set as single-sig users. +Taproot and schnorr also include efficiency improvements for full nodes +such as the ability to batch signature verification. Together, the +improvements lay the groundwork for future potential +upgrades that may improve efficiency, privacy, and fungibility further. + +Activation for taproot is being managed using a variation of BIP9 +versionbits called Speedy Trial (described in BIP341). Taproot's +versionbit is bit 2, and nodes will begin tracking which blocks signal +support for taproot at the beginning of the first retarget period after +taproot’s start date of 24 April 2021. If 90% of blocks within a +2,016-block retarget period (about two weeks) signal support for taproot +prior to the first retarget period beginning after the time of 11 August +2021, the soft fork will be locked in, and taproot will then be active +as of block 709632 (expected in early or mid November). + +Should taproot not be locked in via Speedy Trial activation, it is +expected that a follow-up activation mechanism will be deployed, with +changes to address the reasons the Speedy Trial method failed. + +This release includes the ability to pay taproot addresses, although +payments to such addresses are not secure until taproot activates. +It also includes the ability to relay and mine taproot transactions +after activation. Beyond those two basic capabilities, this release +does not include any code that allows anyone to directly use taproot. +The addition of taproot-related features to Bitcoin Core's wallet is +expected in later releases once taproot activation is assured. + +All users, businesses, and miners are encouraged to upgrade to this +release (or a subsequent compatible release) unless they object to +activation of taproot. If taproot is locked in, then upgrading before +block 709632 is highly recommended to help enforce taproot's new rules +and to avoid the unlikely case of seeing falsely confirmed transactions. + +Miners who want to activate Taproot should preferably use this release +to control their signaling. The `getblocktemplate` RPC results will +automatically be updated to signal once the appropriate start has been +reached and continue signaling until the timeout occurs or taproot +activates. Alternatively, miners may manually start signaling on bit 2 +at any time; if taproot activates, they will need to ensure they update +their nodes before block 709632 or non-upgraded nodes could cause them to mine on +an invalid chain. See the [versionbits +FAQ](https://bitcoincore.org/en/2016/06/08/version-bits-miners-faq/) for +details. + + +For more information about taproot, please see the following resources: + +- Technical specifications + - [BIP340 Schnorr signatures for secp256k1](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki) + - [BIP341 Taproot: SegWit version 1 spending rules](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki) + - [BIP342 Validation of Taproot scripts](https://github.com/bitcoin/bips/blob/master/bip-0342.mediawiki) + +- Popular articles; + - [Taproot Is Coming: What It Is, and How It Will Benefit Bitcoin](https://bitcoinmagazine.com/technical/taproot-coming-what-it-and-how-it-will-benefit-bitcoin) + - [What do Schnorr Signatures Mean for Bitcoin?](https://academy.binance.com/en/articles/what-do-schnorr-signatures-mean-for-bitcoin) + - [The Schnorr Signature & Taproot Softfork Proposal](https://blog.bitmex.com/the-schnorr-signature-taproot-softfork-proposal/) + +- Development history overview + - [Taproot](https://bitcoinops.org/en/topics/taproot/) + - [Schnorr signatures](https://bitcoinops.org/en/topics/schnorr-signatures/) + - [Tapscript](https://bitcoinops.org/en/topics/tapscript/) + - [Soft fork activation](https://bitcoinops.org/en/topics/soft-fork-activation/) + +- Other + - [Questions and answers related to taproot](https://bitcoin.stackexchange.com/questions/tagged/taproot) + - [Taproot review](https://github.com/ajtowns/taproot-review) + +Updated RPCs +------------ + +- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki) + being implemented, behavior for all RPCs that accept addresses is changed when + a native witness version 1 (or higher) is passed. These now require a Bech32m + encoding instead of a Bech32 one, and Bech32m encoding will be used for such + addresses in RPC output as well. No version 1 addresses should be created + for mainnet until consensus rules are adopted that give them meaning + (e.g. through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)). + Once that happens, Bech32m is expected to be used for them, so this shouldn't + affect any production systems, but may be observed on other networks where such + addresses already have meaning (like signet). + +0.21.1 change log +================= + +### Consensus +- #21377 Speedy trial support for versionbits (ajtowns) +- #21686 Speedy trial activation parameters for Taproot (achow101) + +### P2P protocol and network code +- #20852 allow CSubNet of non-IP networks (vasild) +- #21043 Avoid UBSan warning in ProcessMessage(…) (practicalswift) + +### Wallet +- #21166 Introduce DeferredSignatureChecker and have SignatureExtractorClass subclass it (achow101) +- #21083 Avoid requesting fee rates multiple times during coin selection (achow101) + +### RPC and other APIs +- #21201 Disallow sendtoaddress and sendmany when private keys disabled (achow101) + +### Build system +- #21486 link against -lsocket if required for `*ifaddrs` (fanquake) +- #20983 Fix MSVC build after gui#176 (hebasto) + +### Tests and QA +- #21380 Add fuzzing harness for versionbits (ajtowns) +- #20812 fuzz: Bump FuzzedDataProvider.h (MarcoFalke) +- #20740 fuzz: Update FuzzedDataProvider.h from upstream (LLVM) (practicalswift) +- #21446 Update vcpkg checkout commit (sipsorcery) +- #21397 fuzz: Bump FuzzedDataProvider.h (MarcoFalke) +- #21081 Fix the unreachable code at `feature_taproot` (brunoerg) +- #20562 Test that a fully signed tx given to signrawtx is unchanged (achow101) +- #21571 Make sure non-IP peers get discouraged and disconnected (vasild, MarcoFalke) +- #21489 fuzz: cleanups for versionbits fuzzer (ajtowns) + +### Miscellaneous +- #20861 BIP 350: Implement Bech32m and use it for v1+ segwit addresses (sipa) + +### Documentation +- #21384 add signet to bitcoin.conf documentation (jonatack) +- #21342 Remove outdated comment (hebasto) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Aaron Clauson +- Andrew Chow +- Anthony Towns +- Bruno Garcia +- Fabian Jahr +- fanquake +- Hennadii Stepanov +- Jon Atack +- Luke Dashjr +- MarcoFalke +- Pieter Wuille +- practicalswift +- randymcmillan +- Sjors Provoost +- Vasil Dimov +- W. J. van der Laan + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/tor.md b/doc/tor.md index e38ada5bd6..2640a6109b 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -15,6 +15,10 @@ There are several ways to see your local onion address in Bitcoin Core: You may set the `-debug=tor` config logging option to have additional information in the debug log about your Tor configuration. +CLI `-addrinfo` returns the number of addresses known to your node per network +type, including Tor v2 and v3. This is useful to see how many onion addresses +are known to your node for `-onlynet=onion` and how many Tor v3 addresses it +knows when upgrading to current and future Tor releases that support Tor v3 only. ## 1. Run Bitcoin Core behind a Tor proxy diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index 5b7fc776a4..318b48bf2e 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -63,7 +63,12 @@ # Port on which to listen for connections (default: 8333, testnet: 18333, signet: 38333, regtest: 18444) #port= -# Maximum number of inbound+outbound connections. +# Maximum number of inbound + outbound connections (default: 125). This option +# applies only if inbound connections are enabled; otherwise, the number of connections +# will not be more than 11: 8 full-relay connections, 2 block-relay-only ones, and +# occasionally 1 short-lived feeler or extra outbound block-relay-only connection. +# These limits do not apply to connections added manually with the -addnode +# configuration option or the addnode RPC, which have a separate limit of 8 connections. #maxconnections= # @@ -142,8 +147,11 @@ # both prior transactions and several dozen future transactions. #keypool=100 +# Maintain coinstats index used by the gettxoutsetinfo RPC (default: 0). +#coinstatsindex=1 + # Enable pruning to reduce storage requirements by deleting old blocks. -# This mode is incompatible with -txindex and -rescan. +# This mode is incompatible with -txindex, -coinstatsindex and -rescan. # 0 = default (no pruning). # 1 = allows manual pruning via RPC. # >=550 = target to stay under in MiB. diff --git a/src/Makefile.am b/src/Makefile.am index e25e210e1e..972a3e279b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -74,6 +74,7 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_CLI) \ + $(LIBBITCOIN_IPC) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_WALLET_TOOL) \ $(LIBBITCOIN_ZMQ) @@ -152,12 +153,17 @@ BITCOIN_CORE_H = \ i2p.h \ index/base.h \ index/blockfilterindex.h \ + index/coinstatsindex.h \ index/disktxpos.h \ index/txindex.h \ indirectmap.h \ init.h \ + init/common.h \ interfaces/chain.h \ + interfaces/echo.h \ interfaces/handler.h \ + interfaces/init.h \ + interfaces/ipc.h \ interfaces/node.h \ interfaces/wallet.h \ key.h \ @@ -298,6 +304,8 @@ obj/build.h: FORCE "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h +ipc/capnp/libbitcoin_ipc_a-ipc.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) + # server: shared between bitcoind and bitcoin-qt # Contains code accessing mempool and chain state that is meant to be separated # from wallet and gui code (see node/README.md). Shared code should go in @@ -319,6 +327,7 @@ libbitcoin_server_a_SOURCES = \ i2p.cpp \ index/base.cpp \ index/blockfilterindex.cpp \ + index/coinstatsindex.cpp \ index/txindex.cpp \ init.cpp \ mapport.cpp \ @@ -521,6 +530,7 @@ libbitcoin_common_a_SOURCES = \ core_read.cpp \ core_write.cpp \ external_signer.cpp \ + init/common.cpp \ key.cpp \ key_io.cpp \ merkleblock.cpp \ @@ -556,7 +566,9 @@ libbitcoin_util_a_SOURCES = \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ fs.cpp \ + interfaces/echo.cpp \ interfaces/handler.cpp \ + interfaces/init.cpp \ logging.cpp \ random.cpp \ randomenv.cpp \ @@ -632,17 +644,17 @@ bitcoin_bin_ldadd = \ bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS) -bitcoind_SOURCES = $(bitcoin_daemon_sources) +bitcoind_SOURCES = $(bitcoin_daemon_sources) init/bitcoind.cpp bitcoind_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoind_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoind_LDFLAGS = $(bitcoin_bin_ldflags) bitcoind_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) -bitcoin_node_SOURCES = $(bitcoin_daemon_sources) +bitcoin_node_SOURCES = $(bitcoin_daemon_sources) init/bitcoin-node.cpp bitcoin_node_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoin_node_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoin_node_LDFLAGS = $(bitcoin_bin_ldflags) -bitcoin_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) +bitcoin_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) $(LIBBITCOIN_IPC) $(LIBMULTIPROCESS_LIBS) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp @@ -806,6 +818,39 @@ if HARDEN $(AM_V_at) OBJDUMP=$(OBJDUMP) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) endif +libbitcoin_ipc_mpgen_input = \ + ipc/capnp/echo.capnp \ + ipc/capnp/init.capnp +EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) +%.capnp: + +if BUILD_MULTIPROCESS +LIBBITCOIN_IPC=libbitcoin_ipc.a +libbitcoin_ipc_a_SOURCES = \ + ipc/capnp/init-types.h \ + ipc/capnp/protocol.cpp \ + ipc/capnp/protocol.h \ + ipc/exception.h \ + ipc/interfaces.cpp \ + ipc/process.cpp \ + ipc/process.h \ + ipc/protocol.h +libbitcoin_ipc_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +libbitcoin_ipc_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(LIBMULTIPROCESS_CFLAGS) + +include $(MPGEN_PREFIX)/include/mpgen.mk +libbitcoin_ipc_mpgen_output = \ + $(libbitcoin_ipc_mpgen_input:=.c++) \ + $(libbitcoin_ipc_mpgen_input:=.h) \ + $(libbitcoin_ipc_mpgen_input:=.proxy-client.c++) \ + $(libbitcoin_ipc_mpgen_input:=.proxy-server.c++) \ + $(libbitcoin_ipc_mpgen_input:=.proxy-types.c++) \ + $(libbitcoin_ipc_mpgen_input:=.proxy-types.h) \ + $(libbitcoin_ipc_mpgen_input:=.proxy.h) +nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output) +CLEANFILES += $(libbitcoin_ipc_mpgen_output) +endif + if EMBEDDED_LEVELDB include Makefile.crc32c.include include Makefile.leveldb.include diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index a573247afa..74cf6734d6 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -61,6 +61,7 @@ QT_MOC_CPP = \ qt/moc_optionsmodel.cpp \ qt/moc_overviewpage.cpp \ qt/moc_peertablemodel.cpp \ + qt/moc_peertablesortproxy.cpp \ qt/moc_paymentserver.cpp \ qt/moc_psbtoperationsdialog.cpp \ qt/moc_qrimagewidget.cpp \ @@ -134,6 +135,7 @@ BITCOIN_QT_H = \ qt/overviewpage.h \ qt/paymentserver.h \ qt/peertablemodel.h \ + qt/peertablesortproxy.h \ qt/platformstyle.h \ qt/psbtoperationsdialog.h \ qt/qrimagewidget.h \ @@ -232,6 +234,7 @@ BITCOIN_QT_BASE_CPP = \ qt/optionsdialog.cpp \ qt/optionsmodel.cpp \ qt/peertablemodel.cpp \ + qt/peertablesortproxy.cpp \ qt/platformstyle.cpp \ qt/qvalidatedlineedit.cpp \ qt/qvaluecombobox.cpp \ @@ -362,14 +365,12 @@ translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCO $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) @test -f $(RCC) @cp -f $< $(@D)/temp_$(<F) - $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin_locale $(@D)/temp_$(<F) | \ - $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ + $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin_locale $(@D)/temp_$(<F) > $@ @rm $(@D)/temp_$(<F) $(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_FONTS) $(RES_ICONS) $(RES_ANIMATION) @test -f $(RCC) - $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin $< | \ - $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ + $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin $< > $@ CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno qt/temp_bitcoin_locale.qrc @@ -400,12 +401,10 @@ ui_%.h: %.ui $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(UIC) -o $@ $< || (echo "Error creating $@"; false) %.moc: %.cpp - $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(MOC) $(DEFAULT_INCLUDES) $(QT_INCLUDES_UNSUPPRESSED) $(MOC_DEFS) $< | \ - $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ + $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(MOC) $(DEFAULT_INCLUDES) $(QT_INCLUDES_UNSUPPRESSED) $(MOC_DEFS) $< > $@ moc_%.cpp: %.h - $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(MOC) $(DEFAULT_INCLUDES) $(QT_INCLUDES_UNSUPPRESSED) $(MOC_DEFS) $< | \ - $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ + $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(MOC) $(DEFAULT_INCLUDES) $(QT_INCLUDES_UNSUPPRESSED) $(MOC_DEFS) $< > $@ %.qm: %.ts @test -f $(LRELEASE) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 570f011f7a..d55f5e1850 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -80,6 +80,7 @@ BITCOIN_TESTS =\ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ test/coins_tests.cpp \ + test/coinstatsindex_tests.cpp \ test/compilerbug_tests.cpp \ test/compress_tests.cpp \ test/crypto_tests.cpp \ @@ -274,6 +275,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/random.cpp \ test/fuzz/rbf.cpp \ test/fuzz/rolling_bloom_filter.cpp \ + test/fuzz/rpc.cpp \ test/fuzz/script.cpp \ test/fuzz/script_assets_test_minimizer.cpp \ test/fuzz/script_bitcoin_consensus.cpp \ @@ -348,11 +350,6 @@ if EMBEDDED_UNIVALUE $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check endif -if ENABLE_FUZZ_LINK_ALL -all-local: $(FUZZ_BINARY) - bash ./test/fuzz/danger_link_all.sh -endif - %.cpp.test: %.cpp @echo Running tests: `cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1` from $< $(AM_V_at)$(TEST_BINARY) --catch_system_errors=no -l test_suite -t "`cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1`" -- DEBUG_LOG_OUT > $<.log 2>&1 || (cat $<.log && false) diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 67ab02a5b3..aa72981cb4 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -6,6 +6,7 @@ #include <consensus/validation.h> #include <crypto/sha256.h> #include <test/util/mining.h> +#include <test/util/script.h> #include <test/util/setup_common.h> #include <test/util/wallet.h> #include <txmempool.h> @@ -18,23 +19,17 @@ static void AssembleBlock(benchmark::Bench& bench) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); - const std::vector<unsigned char> op_true{OP_TRUE}; CScriptWitness witness; - witness.stack.push_back(op_true); - - uint256 witness_program; - CSHA256().Write(&op_true[0], op_true.size()).Finalize(witness_program.begin()); - - const CScript SCRIPT_PUB{CScript(OP_0) << std::vector<unsigned char>{witness_program.begin(), witness_program.end()}}; + witness.stack.push_back(WITNESS_STACK_ELEM_OP_TRUE); // Collect some loose transactions that spend the coinbases of our mined blocks constexpr size_t NUM_BLOCKS{200}; std::array<CTransactionRef, NUM_BLOCKS - COINBASE_MATURITY + 1> txs; for (size_t b{0}; b < NUM_BLOCKS; ++b) { CMutableTransaction tx; - tx.vin.push_back(MineBlock(test_setup->m_node, SCRIPT_PUB)); + tx.vin.push_back(MineBlock(test_setup->m_node, P2WSH_OP_TRUE)); tx.vin.back().scriptWitness = witness; - tx.vout.emplace_back(1337, SCRIPT_PUB); + tx.vout.emplace_back(1337, P2WSH_OP_TRUE); if (NUM_BLOCKS - b >= COINBASE_MATURITY) txs.at(b) = MakeTransactionRef(tx); } @@ -48,7 +43,7 @@ static void AssembleBlock(benchmark::Bench& bench) } bench.run([&] { - PrepareBlock(test_setup->m_node, SCRIPT_PUB); + PrepareBlock(test_setup->m_node, P2WSH_OP_TRUE); }); } diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 225b8b1ec4..cf9e4fad44 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -12,6 +12,7 @@ #include <compat.h> #include <init.h> #include <interfaces/chain.h> +#include <interfaces/init.h> #include <node/context.h> #include <node/ui_interface.h> #include <noui.h> @@ -104,10 +105,8 @@ int fork_daemon(bool nochdir, bool noclose, TokenPipeEnd& endpoint) #endif -static bool AppInit(int argc, char* argv[]) +static bool AppInit(NodeContext& node, int argc, char* argv[]) { - NodeContext node; - bool fRet = false; util::ThreadSetInternalName("init"); @@ -254,10 +253,18 @@ int main(int argc, char* argv[]) util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); #endif + + NodeContext node; + int exit_status; + std::unique_ptr<interfaces::Init> init = interfaces::MakeNodeInit(node, argc, argv, exit_status); + if (!init) { + return exit_status; + } + SetupEnvironment(); // Connect bitcoind signal handlers noui_connect(); - return (AppInit(argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE); + return (AppInit(node, argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE); } diff --git a/src/chainparamsseeds.h b/src/chainparamsseeds.h index cd7b806621..2e31daea83 100644 --- a/src/chainparamsseeds.h +++ b/src/chainparamsseeds.h @@ -1195,6 +1195,14 @@ static const uint8_t chainparams_seed_main[] = { 0x04,0x20,0x98,0xc6,0x44,0x27,0x90,0x41,0xa6,0x98,0xf9,0x25,0x6c,0x59,0x0f,0x06,0x6d,0x44,0x59,0x0e,0xb2,0x46,0xb0,0xa4,0x37,0x88,0x69,0x8f,0xc1,0x32,0xcd,0x9f,0x15,0xd7,0x20,0x8d, 0x04,0x20,0xaa,0x3a,0x16,0x86,0xea,0x59,0x09,0x04,0x78,0xe5,0x10,0x92,0xe1,0x1d,0xad,0xf7,0x56,0x2b,0xac,0xb0,0x97,0x29,0x63,0x30,0xf4,0x1b,0xcf,0xde,0xf3,0x28,0x0a,0x29,0x20,0x8d, 0x04,0x20,0xbc,0x27,0xae,0x89,0xc1,0x67,0x73,0x0a,0x08,0x02,0xdf,0xb7,0xcc,0x94,0xc7,0x9f,0xf4,0x72,0x7a,0x9b,0x20,0x0c,0x5c,0x11,0x3d,0x22,0xd6,0x13,0x88,0x66,0x74,0xbf,0x20,0x8d, + 0x05,0x20,0xfe,0x97,0xba,0x09,0x2a,0xa4,0x85,0x10,0xa1,0x04,0x7b,0x88,0x7a,0x5a,0x06,0x53,0x71,0x93,0x3b,0xf9,0xa2,0x2f,0xd9,0xe3,0x8f,0xa5,0xa2,0xac,0x1e,0x6c,0x6c,0x8c,0x20,0x8d, + 0x05,0x20,0x17,0x0c,0x56,0xce,0x72,0xa5,0xa0,0xe6,0x23,0x06,0xa3,0xc7,0x08,0x43,0x18,0xee,0x3a,0x46,0x35,0x5d,0x17,0xf6,0x78,0x96,0xa0,0x9c,0x51,0xef,0xbe,0x23,0xfd,0x71,0x20,0x8d, + 0x05,0x20,0x31,0x0f,0x30,0x0b,0x9d,0x70,0x0c,0x7c,0xf7,0x98,0x7e,0x1c,0xf4,0x33,0xdc,0x64,0x17,0xf7,0x00,0x7a,0x0c,0x04,0xb5,0x83,0xfc,0x5f,0xa6,0x52,0x39,0x79,0x63,0x87,0x20,0x8d, + 0x05,0x20,0x3e,0xe3,0xe0,0xa9,0xbc,0xf4,0x2e,0x59,0xd9,0x20,0xee,0xdf,0x74,0x61,0x4d,0x99,0x0c,0x5c,0x15,0x30,0x9b,0x72,0x16,0x79,0x15,0xf4,0x7a,0xca,0x34,0xcc,0x81,0x99,0x20,0x8d, + 0x05,0x20,0x3b,0x42,0x1c,0x25,0xf7,0xbf,0x79,0xed,0x6d,0x7d,0xef,0x65,0x30,0x7d,0xee,0x16,0x37,0x22,0x72,0x43,0x33,0x28,0x40,0xa3,0xaa,0xf4,0x48,0x49,0x67,0xb1,0x4b,0xfd,0x20,0x8d, + 0x05,0x20,0x7a,0x65,0xf7,0x47,0x42,0x9d,0x66,0x42,0x3b,0xb3,0xa7,0x03,0x6c,0x46,0x78,0x19,0x28,0x78,0x1e,0xa3,0x7c,0x67,0x44,0xb7,0x83,0x05,0xe3,0xfe,0xa5,0xe4,0x0a,0x6e,0x20,0x8d, + 0x05,0x20,0xb5,0x83,0x6f,0xb6,0x11,0xd8,0x0e,0xa8,0x57,0xda,0x15,0x20,0x5b,0x1a,0x6d,0x21,0x15,0x5a,0xbd,0xb4,0x17,0x11,0xc2,0xfb,0x0e,0xfc,0xde,0xe8,0x26,0x56,0xa8,0xac,0x20,0x8d, + 0x05,0x20,0xcc,0xaf,0x6c,0x3b,0xd0,0x13,0x76,0x23,0xc3,0x36,0xbb,0x64,0x4a,0x4a,0x06,0x93,0x69,0x6d,0xb0,0x10,0x6e,0x66,0xa4,0x61,0xf8,0x2d,0xe7,0x80,0x72,0x4d,0x53,0x94,0x20,0x8d, }; static const uint8_t chainparams_seed_test[] = { diff --git a/src/compressor.cpp b/src/compressor.cpp index a70306d320..a161c42866 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -52,7 +52,7 @@ static bool IsToPubKey(const CScript& script, CPubKey &pubkey) return false; } -bool CompressScript(const CScript& script, std::vector<unsigned char> &out) +bool CompressScript(const CScript& script, CompressedScript& out) { CKeyID keyID; if (IsToKeyID(script, keyID)) { @@ -92,7 +92,7 @@ unsigned int GetSpecialScriptSize(unsigned int nSize) return 0; } -bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &in) +bool DecompressScript(CScript& script, unsigned int nSize, const CompressedScript& in) { switch(nSize) { case 0x00: @@ -124,7 +124,7 @@ bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<uns unsigned char vch[33] = {}; vch[0] = nSize - 2; memcpy(&vch[1], in.data(), 32); - CPubKey pubkey(&vch[0], &vch[33]); + CPubKey pubkey{vch}; if (!pubkey.Decompress()) return false; assert(pubkey.size() == 65); diff --git a/src/compressor.h b/src/compressor.h index 478bfff0b6..40b2496f06 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -6,14 +6,26 @@ #ifndef BITCOIN_COMPRESSOR_H #define BITCOIN_COMPRESSOR_H +#include <prevector.h> #include <primitives/transaction.h> #include <script/script.h> #include <serialize.h> #include <span.h> -bool CompressScript(const CScript& script, std::vector<unsigned char> &out); +/** + * This saves us from making many heap allocations when serializing + * and deserializing compressed scripts. + * + * This prevector size is determined by the largest .resize() in the + * CompressScript function. The largest compressed script format is a + * compressed public key, which is 33 bytes. + */ +using CompressedScript = prevector<33, unsigned char>; + + +bool CompressScript(const CScript& script, CompressedScript& out); unsigned int GetSpecialScriptSize(unsigned int nSize); -bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &out); +bool DecompressScript(CScript& script, unsigned int nSize, const CompressedScript& in); /** * Compress amount. @@ -51,7 +63,7 @@ struct ScriptCompression template<typename Stream> void Ser(Stream &s, const CScript& script) { - std::vector<unsigned char> compr; + CompressedScript compr; if (CompressScript(script, compr)) { s << MakeSpan(compr); return; @@ -66,7 +78,7 @@ struct ScriptCompression unsigned int nSize = 0; s >> VARINT(nSize); if (nSize < nSpecialScripts) { - std::vector<unsigned char> vch(GetSpecialScriptSize(nSize), 0x00); + CompressedScript vch(GetSpecialScriptSize(nSize), 0x00); s >> MakeSpan(vch); DecompressScript(script, nSize, vch); return; diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp index e5a0d4cb9c..a2b769cd56 100644 --- a/src/crypto/muhash.cpp +++ b/src/crypto/muhash.cpp @@ -341,6 +341,6 @@ MuHash3072& MuHash3072::Insert(Span<const unsigned char> in) noexcept { } MuHash3072& MuHash3072::Remove(Span<const unsigned char> in) noexcept { - m_numerator.Divide(ToNum3072(in)); + m_denominator.Multiply(ToNum3072(in)); return *this; } diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index d7694108f5..3a1086bf4c 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -220,10 +220,9 @@ const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8; */ std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const { - unsigned char buff[OBFUSCATE_KEY_NUM_BYTES]; - GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES); - return std::vector<unsigned char>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]); - + std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES); + GetRandBytes(ret.data(), OBFUSCATE_KEY_NUM_BYTES); + return ret; } bool CDBWrapper::IsEmpty() diff --git a/src/index/base.cpp b/src/index/base.cpp index 9e637c9c6f..357c4fbaf9 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -13,7 +13,7 @@ #include <validation.h> // For g_chainman #include <warnings.h> -constexpr char DB_BEST_BLOCK = 'B'; +constexpr uint8_t DB_BEST_BLOCK{'B'}; constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds diff --git a/src/index/base.h b/src/index/base.h index ac3c429a5a..d887620524 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -81,6 +81,8 @@ protected: void ChainStateFlushed(const CBlockLocator& locator) override; + const CBlockIndex* CurrentIndex() { return m_best_block_index.load(); }; + /// Initialize internal state from the database and block index. virtual bool Init(); diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 154d7a7027..b82b9915d5 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -24,9 +24,9 @@ * as big-endian so that sequential reads of filters by height are fast. * Keys for the hash index have the type [DB_BLOCK_HASH, uint256]. */ -constexpr char DB_BLOCK_HASH = 's'; -constexpr char DB_BLOCK_HEIGHT = 't'; -constexpr char DB_FILTER_POS = 'P'; +constexpr uint8_t DB_BLOCK_HASH{'s'}; +constexpr uint8_t DB_BLOCK_HEIGHT{'t'}; +constexpr uint8_t DB_FILTER_POS{'P'}; constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB /** The pre-allocation chunk size for fltr?????.dat files */ @@ -63,7 +63,7 @@ struct DBHeightKey { template<typename Stream> void Unserialize(Stream& s) { - char prefix = ser_readdata8(s); + const uint8_t prefix{ser_readdata8(s)}; if (prefix != DB_BLOCK_HEIGHT) { throw std::ios_base::failure("Invalid format for block filter index DB height key"); } @@ -77,7 +77,7 @@ struct DBHashKey { explicit DBHashKey(const uint256& hash_in) : hash(hash_in) {} SERIALIZE_METHODS(DBHashKey, obj) { - char prefix = DB_BLOCK_HASH; + uint8_t prefix{DB_BLOCK_HASH}; READWRITE(prefix); if (prefix != DB_BLOCK_HASH) { throw std::ios_base::failure("Invalid format for block filter index DB hash key"); @@ -149,7 +149,7 @@ bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& f } uint256 block_hash; - std::vector<unsigned char> encoded_filter; + std::vector<uint8_t> encoded_filter; try { filein >> block_hash >> encoded_filter; filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter)); diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp new file mode 100644 index 0000000000..7c8b2b186e --- /dev/null +++ b/src/index/coinstatsindex.cpp @@ -0,0 +1,472 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <coins.h> +#include <crypto/muhash.h> +#include <index/coinstatsindex.h> +#include <node/blockstorage.h> +#include <serialize.h> +#include <txdb.h> +#include <undo.h> +#include <validation.h> + +static constexpr uint8_t DB_BLOCK_HASH{'s'}; +static constexpr uint8_t DB_BLOCK_HEIGHT{'t'}; +static constexpr uint8_t DB_MUHASH{'M'}; + +namespace { + +struct DBVal { + uint256 muhash; + uint64_t transaction_output_count; + uint64_t bogo_size; + CAmount total_amount; + CAmount total_subsidy; + CAmount block_unspendable_amount; + CAmount block_prevout_spent_amount; + CAmount block_new_outputs_ex_coinbase_amount; + CAmount block_coinbase_amount; + CAmount unspendables_genesis_block; + CAmount unspendables_bip30; + CAmount unspendables_scripts; + CAmount unspendables_unclaimed_rewards; + + SERIALIZE_METHODS(DBVal, obj) + { + READWRITE(obj.muhash); + READWRITE(obj.transaction_output_count); + READWRITE(obj.bogo_size); + READWRITE(obj.total_amount); + READWRITE(obj.total_subsidy); + READWRITE(obj.block_unspendable_amount); + READWRITE(obj.block_prevout_spent_amount); + READWRITE(obj.block_new_outputs_ex_coinbase_amount); + READWRITE(obj.block_coinbase_amount); + READWRITE(obj.unspendables_genesis_block); + READWRITE(obj.unspendables_bip30); + READWRITE(obj.unspendables_scripts); + READWRITE(obj.unspendables_unclaimed_rewards); + } +}; + +struct DBHeightKey { + int height; + + explicit DBHeightKey(int height_in) : height(height_in) {} + + template <typename Stream> + void Serialize(Stream& s) const + { + ser_writedata8(s, DB_BLOCK_HEIGHT); + ser_writedata32be(s, height); + } + + template <typename Stream> + void Unserialize(Stream& s) + { + const uint8_t prefix{ser_readdata8(s)}; + if (prefix != DB_BLOCK_HEIGHT) { + throw std::ios_base::failure("Invalid format for coinstatsindex DB height key"); + } + height = ser_readdata32be(s); + } +}; + +struct DBHashKey { + uint256 block_hash; + + explicit DBHashKey(const uint256& hash_in) : block_hash(hash_in) {} + + SERIALIZE_METHODS(DBHashKey, obj) + { + uint8_t prefix{DB_BLOCK_HASH}; + READWRITE(prefix); + if (prefix != DB_BLOCK_HASH) { + throw std::ios_base::failure("Invalid format for coinstatsindex DB hash key"); + } + + READWRITE(obj.block_hash); + } +}; + +}; // namespace + +std::unique_ptr<CoinStatsIndex> g_coin_stats_index; + +CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe) +{ + fs::path path{GetDataDir() / "indexes" / "coinstats"}; + fs::create_directories(path); + + m_db = std::make_unique<CoinStatsIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe); +} + +bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; + m_total_subsidy += block_subsidy; + + // Ignore genesis block + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + std::pair<uint256, DBVal> read_out; + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; + if (read_out.first != expected_block_hash) { + if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + } + + // TODO: Deduplicate BIP30 related code + bool is_bip30_block{(pindex->nHeight == 91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (pindex->nHeight == 91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))}; + + // Add the new utxos created from the block + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx{block.vtx.at(i)}; + + // Skip duplicate txid coinbase transactions (BIP30). + if (is_bip30_block && tx->IsCoinBase()) { + m_block_unspendable_amount += block_subsidy; + m_unspendables_bip30 += block_subsidy; + continue; + } + + for (size_t j = 0; j < tx->vout.size(); ++j) { + const CTxOut& out{tx->vout[j]}; + Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; + COutPoint outpoint{tx->GetHash(), static_cast<uint32_t>(j)}; + + // Skip unspendable coins + if (coin.out.scriptPubKey.IsUnspendable()) { + m_block_unspendable_amount += coin.out.nValue; + m_unspendables_scripts += coin.out.nValue; + continue; + } + + m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + + if (tx->IsCoinBase()) { + m_block_coinbase_amount += coin.out.nValue; + } else { + m_block_new_outputs_ex_coinbase_amount += coin.out.nValue; + } + + ++m_transaction_output_count; + m_total_amount += coin.out.nValue; + m_bogo_size += GetBogoSize(coin.out.scriptPubKey); + } + + // The coinbase tx has no undo data since no former output is spent + if (!tx->IsCoinBase()) { + const auto& tx_undo{block_undo.vtxundo.at(i - 1)}; + + for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { + Coin coin{tx_undo.vprevout[j]}; + COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n}; + + m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); + + m_block_prevout_spent_amount += coin.out.nValue; + + --m_transaction_output_count; + m_total_amount -= coin.out.nValue; + m_bogo_size -= GetBogoSize(coin.out.scriptPubKey); + } + } + } + } else { + // genesis block + m_block_unspendable_amount += block_subsidy; + m_unspendables_genesis_block += block_subsidy; + } + + // If spent prevouts + block subsidy are still a higher amount than + // new outputs + coinbase + current unspendable amount this means + // the miner did not claim the full block reward. Unclaimed block + // rewards are also unspendable. + const CAmount unclaimed_rewards{(m_block_prevout_spent_amount + m_total_subsidy) - (m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount)}; + m_block_unspendable_amount += unclaimed_rewards; + m_unspendables_unclaimed_rewards += unclaimed_rewards; + + std::pair<uint256, DBVal> value; + value.first = pindex->GetBlockHash(); + value.second.transaction_output_count = m_transaction_output_count; + value.second.bogo_size = m_bogo_size; + value.second.total_amount = m_total_amount; + value.second.total_subsidy = m_total_subsidy; + value.second.block_unspendable_amount = m_block_unspendable_amount; + value.second.block_prevout_spent_amount = m_block_prevout_spent_amount; + value.second.block_new_outputs_ex_coinbase_amount = m_block_new_outputs_ex_coinbase_amount; + value.second.block_coinbase_amount = m_block_coinbase_amount; + value.second.unspendables_genesis_block = m_unspendables_genesis_block; + value.second.unspendables_bip30 = m_unspendables_bip30; + value.second.unspendables_scripts = m_unspendables_scripts; + value.second.unspendables_unclaimed_rewards = m_unspendables_unclaimed_rewards; + + uint256 out; + m_muhash.Finalize(out); + value.second.muhash = out; + + return m_db->Write(DBHeightKey(pindex->nHeight), value) && m_db->Write(DB_MUHASH, m_muhash); +} + +static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, + const std::string& index_name, + int start_height, int stop_height) +{ + DBHeightKey key{start_height}; + db_it.Seek(key); + + for (int height = start_height; height <= stop_height; ++height) { + if (!db_it.GetKey(key) || key.height != height) { + return error("%s: unexpected key in %s: expected (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + std::pair<uint256, DBVal> value; + if (!db_it.GetValue(value)) { + return error("%s: unable to read value in %s at key (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + batch.Write(DBHashKey(value.first), std::move(value.second)); + + db_it.Next(); + } + return true; +} + +bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +{ + assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + + CDBBatch batch(*m_db); + std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); + + // During a reorg, we need to copy all hash digests for blocks that are + // getting disconnected from the height index to the hash index so we can + // still find them when the height index entries are overwritten. + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + return false; + } + + if (!m_db->WriteBatch(batch)) return false; + + { + LOCK(cs_main); + CBlockIndex* iter_tip{g_chainman.m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; + const auto& consensus_params{Params().GetConsensus()}; + + do { + CBlock block; + + if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) { + return error("%s: Failed to read block %s from disk", + __func__, iter_tip->GetBlockHash().ToString()); + } + + ReverseBlock(block, iter_tip); + + iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1); + } while (new_tip != iter_tip); + } + + return BaseIndex::Rewind(current_tip, new_tip); +} + +static bool LookUpOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) +{ + // First check if the result is stored under the height index and the value + // there matches the block hash. This should be the case if the block is on + // the active chain. + std::pair<uint256, DBVal> read_out; + if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { + return false; + } + if (read_out.first == block_index->GetBlockHash()) { + result = std::move(read_out.second); + return true; + } + + // If value at the height index corresponds to an different block, the + // result will be stored in the hash index. + return db.Read(DBHashKey(block_index->GetBlockHash()), result); +} + +bool CoinStatsIndex::LookUpStats(const CBlockIndex* block_index, CCoinsStats& coins_stats) const +{ + DBVal entry; + if (!LookUpOne(*m_db, block_index, entry)) { + return false; + } + + coins_stats.hashSerialized = entry.muhash; + coins_stats.nTransactionOutputs = entry.transaction_output_count; + coins_stats.nBogoSize = entry.bogo_size; + coins_stats.nTotalAmount = entry.total_amount; + coins_stats.total_subsidy = entry.total_subsidy; + coins_stats.block_unspendable_amount = entry.block_unspendable_amount; + coins_stats.block_prevout_spent_amount = entry.block_prevout_spent_amount; + coins_stats.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; + coins_stats.block_coinbase_amount = entry.block_coinbase_amount; + coins_stats.unspendables_genesis_block = entry.unspendables_genesis_block; + coins_stats.unspendables_bip30 = entry.unspendables_bip30; + coins_stats.unspendables_scripts = entry.unspendables_scripts; + coins_stats.unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards; + + return true; +} + +bool CoinStatsIndex::Init() +{ + if (!m_db->Read(DB_MUHASH, m_muhash)) { + // Check that the cause of the read failure is that the key does not + // exist. Any other errors indicate database corruption or a disk + // failure, and starting the index would cause further corruption. + if (m_db->Exists(DB_MUHASH)) { + return error("%s: Cannot read current %s state; index may be corrupted", + __func__, GetName()); + } + } + + if (BaseIndex::Init()) { + const CBlockIndex* pindex{CurrentIndex()}; + + if (pindex) { + DBVal entry; + if (!LookUpOne(*m_db, pindex, entry)) { + return false; + } + + m_transaction_output_count = entry.transaction_output_count; + m_bogo_size = entry.bogo_size; + m_total_amount = entry.total_amount; + m_total_subsidy = entry.total_subsidy; + m_block_unspendable_amount = entry.block_unspendable_amount; + m_block_prevout_spent_amount = entry.block_prevout_spent_amount; + m_block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; + m_block_coinbase_amount = entry.block_coinbase_amount; + m_unspendables_genesis_block = entry.unspendables_genesis_block; + m_unspendables_bip30 = entry.unspendables_bip30; + m_unspendables_scripts = entry.unspendables_scripts; + m_unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards; + } + + return true; + } + + return false; +} + +// Reverse a single block as part of a reorg +bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + std::pair<uint256, DBVal> read_out; + + const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; + m_total_subsidy -= block_subsidy; + + // Ignore genesis block + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; + if (read_out.first != expected_block_hash) { + if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + } + } + + // Remove the new UTXOs that were created from the block + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx{block.vtx.at(i)}; + + for (size_t j = 0; j < tx->vout.size(); ++j) { + const CTxOut& out{tx->vout[j]}; + COutPoint outpoint{tx->GetHash(), static_cast<uint32_t>(j)}; + Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; + + // Skip unspendable coins + if (coin.out.scriptPubKey.IsUnspendable()) { + m_block_unspendable_amount -= coin.out.nValue; + m_unspendables_scripts -= coin.out.nValue; + continue; + } + + m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); + + if (tx->IsCoinBase()) { + m_block_coinbase_amount -= coin.out.nValue; + } else { + m_block_new_outputs_ex_coinbase_amount -= coin.out.nValue; + } + + --m_transaction_output_count; + m_total_amount -= coin.out.nValue; + m_bogo_size -= GetBogoSize(coin.out.scriptPubKey); + } + + // The coinbase tx has no undo data since no former output is spent + if (!tx->IsCoinBase()) { + const auto& tx_undo{block_undo.vtxundo.at(i - 1)}; + + for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { + Coin coin{tx_undo.vprevout[j]}; + COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n}; + + m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + + m_block_prevout_spent_amount -= coin.out.nValue; + + m_transaction_output_count++; + m_total_amount += coin.out.nValue; + m_bogo_size += GetBogoSize(coin.out.scriptPubKey); + } + } + } + + const CAmount unclaimed_rewards{(m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount) - (m_block_prevout_spent_amount + m_total_subsidy)}; + m_block_unspendable_amount -= unclaimed_rewards; + m_unspendables_unclaimed_rewards -= unclaimed_rewards; + + // Check that the rolled back internal values are consistent with the DB read out + uint256 out; + m_muhash.Finalize(out); + Assert(read_out.second.muhash == out); + + Assert(m_transaction_output_count == read_out.second.transaction_output_count); + Assert(m_total_amount == read_out.second.total_amount); + Assert(m_bogo_size == read_out.second.bogo_size); + Assert(m_total_subsidy == read_out.second.total_subsidy); + Assert(m_block_unspendable_amount == read_out.second.block_unspendable_amount); + Assert(m_block_prevout_spent_amount == read_out.second.block_prevout_spent_amount); + Assert(m_block_new_outputs_ex_coinbase_amount == read_out.second.block_new_outputs_ex_coinbase_amount); + Assert(m_block_coinbase_amount == read_out.second.block_coinbase_amount); + Assert(m_unspendables_genesis_block == read_out.second.unspendables_genesis_block); + Assert(m_unspendables_bip30 == read_out.second.unspendables_bip30); + Assert(m_unspendables_scripts == read_out.second.unspendables_scripts); + Assert(m_unspendables_unclaimed_rewards == read_out.second.unspendables_unclaimed_rewards); + + return m_db->Write(DB_MUHASH, m_muhash); +} diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h new file mode 100644 index 0000000000..6149f9b4b3 --- /dev/null +++ b/src/index/coinstatsindex.h @@ -0,0 +1,61 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_COINSTATSINDEX_H +#define BITCOIN_INDEX_COINSTATSINDEX_H + +#include <chain.h> +#include <crypto/muhash.h> +#include <flatfile.h> +#include <index/base.h> +#include <node/coinstats.h> + +/** + * CoinStatsIndex maintains statistics on the UTXO set. + */ +class CoinStatsIndex final : public BaseIndex +{ +private: + std::string m_name; + std::unique_ptr<BaseIndex::DB> m_db; + + MuHash3072 m_muhash; + uint64_t m_transaction_output_count{0}; + uint64_t m_bogo_size{0}; + CAmount m_total_amount{0}; + CAmount m_total_subsidy{0}; + CAmount m_block_unspendable_amount{0}; + CAmount m_block_prevout_spent_amount{0}; + CAmount m_block_new_outputs_ex_coinbase_amount{0}; + CAmount m_block_coinbase_amount{0}; + CAmount m_unspendables_genesis_block{0}; + CAmount m_unspendables_bip30{0}; + CAmount m_unspendables_scripts{0}; + CAmount m_unspendables_unclaimed_rewards{0}; + + bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); + +protected: + bool Init() override; + + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + + bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + + BaseIndex::DB& GetDB() const override { return *m_db; } + + const char* GetName() const override { return "coinstatsindex"; } + +public: + // Constructs the index, which becomes available to be queried. + explicit CoinStatsIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + // Look up stats for a specific block using CBlockIndex + bool LookUpStats(const CBlockIndex* block_index, CCoinsStats& coins_stats) const; +}; + +/// The global UTXO set hash object. +extern std::unique_ptr<CoinStatsIndex> g_coin_stats_index; + +#endif // BITCOIN_INDEX_COINSTATSINDEX_H diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index f41985c344..3feefe8619 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -4,20 +4,20 @@ #include <index/disktxpos.h> #include <index/txindex.h> +#include <node/blockstorage.h> #include <node/ui_interface.h> #include <shutdown.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> -constexpr char DB_BEST_BLOCK = 'B'; -constexpr char DB_TXINDEX = 't'; -constexpr char DB_TXINDEX_BLOCK = 'T'; +constexpr uint8_t DB_BEST_BLOCK{'B'}; +constexpr uint8_t DB_TXINDEX{'t'}; +constexpr uint8_t DB_TXINDEX_BLOCK{'T'}; std::unique_ptr<TxIndex> g_txindex; - /** Access to the txindex database (indexes/txindex/) */ class TxIndex::DB : public BaseIndex::DB { @@ -60,8 +60,8 @@ bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_ */ static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb, CDBBatch& batch_newdb, CDBBatch& batch_olddb, - const std::pair<unsigned char, uint256>& begin_key, - const std::pair<unsigned char, uint256>& end_key) + const std::pair<uint8_t, uint256>& begin_key, + const std::pair<uint8_t, uint256>& end_key) { // Sync new DB changes to disk before deleting from old DB. newdb.WriteBatch(batch_newdb, /*fSync=*/ true); @@ -113,9 +113,9 @@ bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& CDBBatch batch_newdb(*this); CDBBatch batch_olddb(block_tree_db); - std::pair<unsigned char, uint256> key; - std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()}; - std::pair<unsigned char, uint256> prev_key = begin_key; + std::pair<uint8_t, uint256> key; + std::pair<uint8_t, uint256> begin_key{DB_TXINDEX, uint256()}; + std::pair<uint8_t, uint256> prev_key = begin_key; bool interrupted = false; std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator()); diff --git a/src/init.cpp b/src/init.cpp index 741e70f748..dbc4ea30c5 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -21,10 +21,11 @@ #include <httprpc.h> #include <httpserver.h> #include <index/blockfilterindex.h> +#include <index/coinstatsindex.h> #include <index/txindex.h> +#include <init/common.h> #include <interfaces/chain.h> #include <interfaces/node.h> -#include <key.h> #include <mapport.h> #include <miner.h> #include <net.h> @@ -151,8 +152,6 @@ static fs::path GetPidFile(const ArgsManager& args) // shutdown thing. // -static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; - void Interrupt(NodeContext& node) { InterruptHTTPServer(); @@ -167,6 +166,9 @@ void Interrupt(NodeContext& node) g_txindex->Interrupt(); } ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); }); + if (g_coin_stats_index) { + g_coin_stats_index->Interrupt(); + } } void Shutdown(NodeContext& node) @@ -196,20 +198,7 @@ void Shutdown(NodeContext& node) // Because these depend on each-other, we make sure that neither can be // using the other before destroying them. if (node.peerman) UnregisterValidationInterface(node.peerman.get()); - // Follow the lock order requirements: - // * CheckForStaleTipAndEvictPeers locks cs_main before indirectly calling GetExtraFullOutboundCount - // which locks cs_vNodes. - // * ProcessMessage locks cs_main and g_cs_orphans before indirectly calling ForEachNode which - // locks cs_vNodes. - // * CConnman::Stop calls DeleteNode, which calls FinalizeNode, which locks cs_main and calls - // EraseOrphansFor, which locks g_cs_orphans. - // - // Thus the implicit locking order requirement is: (1) cs_main, (2) g_cs_orphans, (3) cs_vNodes. - if (node.connman) { - node.connman->StopThreads(); - LOCK2(::cs_main, ::g_cs_orphans); - node.connman->StopNodes(); - } + if (node.connman) node.connman->Stop(); StopTorControl(); @@ -252,6 +241,10 @@ void Shutdown(NodeContext& node) g_txindex->Stop(); g_txindex.reset(); } + if (g_coin_stats_index) { + g_coin_stats_index->Stop(); + g_coin_stats_index.reset(); + } ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); DestroyAllBlockFilterIndexes(); @@ -286,8 +279,7 @@ void Shutdown(NodeContext& node) node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - globalVerifyHandle.reset(); - ECC_Stop(); + init::UnsetGlobals(); node.mempool.reset(); node.fee_estimator.reset(); node.chainman = nullptr; @@ -363,6 +355,8 @@ void SetupServerArgs(NodeContext& node) SetupHelpOptions(argsman); argsman.AddArg("-help-debug", "Print help message with debugging options and exit", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); // server-only for now + init::AddLoggingArgs(argsman); + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); const auto signetBaseParams = CreateBaseChainParams(CBaseChainParams::SIGNET); @@ -390,11 +384,11 @@ void SetupServerArgs(NodeContext& node) #endif argsman.AddArg("-blockreconstructionextratxn=<n>", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless the peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-coinstatsindex", strprintf("Maintain coinstats index used by the gettxoutsetinfo RPC (default: %u)", DEFAULT_COINSTATSINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -406,7 +400,7 @@ void SetupServerArgs(NodeContext& node) -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. " + argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -coinstatsindex and -rescan. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -426,7 +420,7 @@ void SetupServerArgs(NodeContext& node) " If <type> is not supplied or if <type> = 1, indexes for all known types are enabled.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-addnode=<ip>", strprintf("Add a node to connect to and attempt to keep the connection open (see the addnode RPC help for more info). This option can be specified multiple times to add multiple nodes; connections are limited to %u at a time and are counted separately from the -maxconnections limit.", MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-asmap=<file>", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bantime=<n>", strprintf("Default duration (in seconds) of manually configured bans (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bind=<addr>[:<port>][=onion]", strprintf("Bind to given address and always listen on it (default: 0.0.0.0). Use [host]:port notation for IPv6. Append =onion to tag any incoming connections to that address and port as incoming Tor connections (default: 127.0.0.1:%u=onion, testnet: 127.0.0.1:%u=onion, signet: 127.0.0.1:%u=onion, regtest: 127.0.0.1:%u=onion)", defaultBaseParams->OnionServiceTargetPort(), testnetBaseParams->OnionServiceTargetPort(), signetBaseParams->OnionServiceTargetPort(), regtestBaseParams->OnionServiceTargetPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); @@ -439,7 +433,7 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listen", "Accept connections from outside (default: 1 if no -proxy or -connect)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listenonion", strprintf("Automatically create Tor onion service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u)", DEFAULT_MAX_PEER_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -522,25 +516,10 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_BOOL | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " - "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ". This option can be specified multiple times to output multiple categories.", - ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except the specified category. This option can be specified multiple times to exclude multiple categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); -#ifdef HAVE_THREAD_LOCAL - argsman.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (only available on platforms supporting thread_local) (default: %u)", DEFAULT_LOGTHREADNAMES), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); -#else - hidden_args.emplace_back("-logthreadnames"); -#endif - argsman.AddArg("-logsourcelocations", strprintf("Prepend debug output with name of the originating source location (source file, line number and function name) (default: %u)", DEFAULT_LOGSOURCELOCATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -daemon. To disable logging to file, set -nodebuglogfile)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-shrinkdebugfile", "Shrink debug.log file on client startup (default: 1 when no -debug)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-uacomment=<cmt>", "Append comment to the user agent string", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); SetupChainParamsBaseOptions(argsman); @@ -623,47 +602,6 @@ static void BlockNotifyGenesisWait(const CBlockIndex* pBlockIndex) } } -// If we're using -prune with -reindex, then delete block files that will be ignored by the -// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile -// is missing, do the same here to delete any later block files after a gap. Also delete all -// rev files since they'll be rewritten by the reindex anyway. This ensures that vinfoBlockFile -// is in sync with what's actually on disk by the time we start downloading, so that pruning -// works correctly. -static void CleanupBlockRevFiles() -{ - std::map<std::string, fs::path> mapBlockFiles; - - // Glob all blk?????.dat and rev?????.dat files from the blocks directory. - // Remove the rev files immediately and insert the blk file paths into an - // ordered map keyed by block file index. - LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); - fs::path blocksdir = gArgs.GetBlocksDirPath(); - for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { - if (fs::is_regular_file(*it) && - it->path().filename().string().length() == 12 && - it->path().filename().string().substr(8,4) == ".dat") - { - if (it->path().filename().string().substr(0,3) == "blk") - mapBlockFiles[it->path().filename().string().substr(3,5)] = it->path(); - else if (it->path().filename().string().substr(0,3) == "rev") - remove(it->path()); - } - } - - // Remove all block files that aren't part of a contiguous set starting at - // zero by walking the ordered map (keys are block file indices) by - // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) - // start removing block files. - int nContigCounter = 0; - for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) { - if (atoi(item.first) == nContigCounter) { - nContigCounter++; - continue; - } - remove(item.second); - } -} - #if HAVE_SYSTEM static void StartupNotify(const ArgsManager& args) { @@ -675,30 +613,6 @@ static void StartupNotify(const ArgsManager& args) } #endif -/** Sanity checks - * Ensure that Bitcoin is running in a usable environment with all - * necessary library support. - */ -static bool InitSanityCheck() -{ - if (!ECC_InitSanityCheck()) { - return InitError(Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")); - } - - if (!glibcxx_sanity_test()) - return false; - - if (!Random_SanityCheck()) { - return InitError(Untranslated("OS cryptographic RNG sanity check failure. Aborting.")); - } - - if (!ChronoSanityCheck()) { - return InitError(Untranslated("Clock epoch mismatch. Aborting.")); - } - - return true; -} - static bool AppInitServers(NodeContext& node) { const ArgsManager& args = *Assert(node.args); @@ -796,25 +710,8 @@ void InitParameterInteraction(ArgsManager& args) */ void InitLogging(const ArgsManager& args) { - LogInstance().m_print_to_file = !args.IsArgNegated("-debuglogfile"); - LogInstance().m_file_path = AbsPathForConfigVal(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); - LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", !args.GetBoolArg("-daemon", false)); - LogInstance().m_log_timestamps = args.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); - LogInstance().m_log_time_micros = args.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); -#ifdef HAVE_THREAD_LOCAL - LogInstance().m_log_threadnames = args.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); -#endif - LogInstance().m_log_sourcelocations = args.GetBoolArg("-logsourcelocations", DEFAULT_LOGSOURCELOCATIONS); - - fLogIPs = args.GetBoolArg("-logips", DEFAULT_LOGIPS); - - std::string version_string = FormatFullVersion(); -#ifdef DEBUG - version_string += " (debug build)"; -#else - version_string += " (release build)"; -#endif - LogPrintf(PACKAGE_NAME " version %s\n", version_string); + init::SetLoggingOptions(args); + init::LogPackageVersion(); } namespace { // Variables internal to initialization process only @@ -947,10 +844,12 @@ bool AppInitParameterInteraction(const ArgsManager& args) nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS); } - // if using block pruning, then disallow txindex + // if using block pruning, then disallow txindex and coinstatsindex if (args.GetArg("-prune", 0)) { if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); + if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) + return InitError(_("Prune mode is incompatible with -coinstatsindex.")); } // -bind and -whitebind can't be set when not listening @@ -982,26 +881,7 @@ bool AppInitParameterInteraction(const ArgsManager& args) InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags - if (args.IsArgSet("-debug")) { - // Special-case: if -debug=0/-nodebug is set, turn off debugging messages - const std::vector<std::string> categories = args.GetArgs("-debug"); - - if (std::none_of(categories.begin(), categories.end(), - [](std::string cat){return cat == "0" || cat == "none";})) { - for (const auto& cat : categories) { - if (!LogInstance().EnableCategory(cat)) { - InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)); - } - } - } - } - - // Now remove the logging categories which were explicitly excluded - for (const std::string& cat : args.GetArgs("-debugexclude")) { - if (!LogInstance().DisableCategory(cat)) { - InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); - } - } + init::SetLoggingCategories(args); fCheckBlockIndex = args.GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); fCheckpointsEnabled = args.GetBoolArg("-checkpoints", DEFAULT_CHECKPOINTS_ENABLED); @@ -1148,16 +1028,11 @@ bool AppInitSanityChecks() { // ********************************************************* Step 4: sanity checks - // Initialize elliptic curve code - std::string sha256_algo = SHA256AutoDetect(); - LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); - RandomInit(); - ECC_Start(); - globalVerifyHandle.reset(new ECCVerifyHandle()); + init::SetGlobals(); - // Sanity check - if (!InitSanityCheck()) + if (!init::SanityChecks()) { return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); + } // Probe the data directory lock to give an early error message, if possible // We cannot hold the data directory lock here, as the forking for daemon() hasn't yet happened, @@ -1197,38 +1072,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Detailed error printed inside CreatePidFile(). return false; } - if (LogInstance().m_print_to_file) { - if (args.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { - // Do this first since it both loads a bunch of debug.log into memory, - // and because this needs to happen before any other debug.log printing - LogInstance().ShrinkDebugFile(); - } - } - if (!LogInstance().StartLogging()) { - return InitError(strprintf(Untranslated("Could not open debug log file %s"), - LogInstance().m_file_path.string())); - } - - if (!LogInstance().m_log_timestamps) - LogPrintf("Startup time: %s\n", FormatISO8601DateTime(GetTime())); - LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); - LogPrintf("Using data directory %s\n", GetDataDir().string()); - - // Only log conf file usage message if conf file actually exists. - fs::path config_file_path = GetConfigFile(args.GetArg("-conf", BITCOIN_CONF_FILENAME)); - if (fs::exists(config_file_path)) { - LogPrintf("Config file: %s\n", config_file_path.string()); - } else if (args.IsArgSet("-conf")) { - // Warn if no conf file exists at path provided by user - InitWarning(strprintf(_("The specified config file %s does not exist"), config_file_path.string())); - } else { - // Not categorizing as "Warning" because it's the default behavior - LogPrintf("Config file: %s (not found, skipping)\n", config_file_path.string()); + if (!init::StartLogging(args)) { + // Detailed error printed inside StartLogging(). + return false; } - // Log the config arguments to debug.log - args.LogArgs(); - LogPrintf("Using at most %i automatic connections (%i file descriptors available)\n", nMaxConnections, nFD); // Warn about relative -datadir path. @@ -1492,7 +1340,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) bool fLoaded = false; while (!fLoaded && !ShutdownRequested()) { - bool fReset = fReindex; + const bool fReset = fReindex; auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); }; @@ -1613,29 +1461,17 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) break; } - bool failed_rewind{false}; - // Can't hold cs_main while calling RewindBlockIndex, so retrieve the relevant - // chainstates beforehand. - for (CChainState* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { - if (!fReset) { - // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. - // It both disconnects blocks based on the chainstate, and drops block data in - // BlockIndex() based on lack of available witness data. - uiInterface.InitMessage(_("Rewinding blocks...").translated); - if (!chainstate->RewindBlockIndex(chainparams)) { - strLoadError = _( - "Unable to rewind the database to a pre-fork state. " - "You will need to redownload the blockchain"); - failed_rewind = true; - break; // out of the per-chainstate loop - } + if (!fReset) { + LOCK(cs_main); + auto chainstates{chainman.GetAll()}; + if (std::any_of(chainstates.begin(), chainstates.end(), + [&chainparams](const CChainState* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(chainparams); })) { + strLoadError = strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."), + chainparams.GetConsensus().SegwitHeight); + break; } } - if (failed_rewind) { - break; // out of the chainstate activation do-while - } - bool failed_verification = false; try { @@ -1659,11 +1495,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) break; } - // Only verify the DB of the active chainstate. This is fixed in later - // work when we allow VerifyDB to be parameterized by chainstate. - if (&::ChainstateActive() == chainstate && - !CVerifyDB().VerifyDB( - chainparams, *chainstate, &chainstate->CoinsDB(), + if (!CVerifyDB().VerifyDB( + *chainstate, chainparams, chainstate->CoinsDB(), args.GetArg("-checklevel", DEFAULT_CHECKLEVEL), args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { strLoadError = _("Corrupted block database detected"); @@ -1724,6 +1557,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) GetBlockFilterIndex(filter_type)->Start(); } + if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { + g_coin_stats_index = std::make_unique<CoinStatsIndex>(/* cache size */ 0, false, fReindex); + g_coin_stats_index->Start(); + } + // ********************************************************* Step 9: load wallet for (const auto& client : node.chain_clients) { if (!client->load()) { diff --git a/src/init/bitcoin-node.cpp b/src/init/bitcoin-node.cpp new file mode 100644 index 0000000000..49684ede83 --- /dev/null +++ b/src/init/bitcoin-node.cpp @@ -0,0 +1,45 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/echo.h> +#include <interfaces/init.h> +#include <interfaces/ipc.h> +#include <node/context.h> + +#include <memory> + +namespace init { +namespace { +const char* EXE_NAME = "bitcoin-node"; + +class BitcoinNodeInit : public interfaces::Init +{ +public: + BitcoinNodeInit(NodeContext& node, const char* arg0) + : m_node(node), + m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this)) + { + m_node.init = this; + } + std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); } + interfaces::Ipc* ipc() override { return m_ipc.get(); } + NodeContext& m_node; + std::unique_ptr<interfaces::Ipc> m_ipc; +}; +} // namespace +} // namespace init + +namespace interfaces { +std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status) +{ + auto init = std::make_unique<init::BitcoinNodeInit>(node, argc > 0 ? argv[0] : ""); + // Check if bitcoin-node is being invoked as an IPC server. If so, then + // bypass normal execution and just respond to requests over the IPC + // channel and return null. + if (init->m_ipc->startSpawnedProcess(argc, argv, exit_status)) { + return nullptr; + } + return init; +} +} // namespace interfaces diff --git a/src/init/bitcoind.cpp b/src/init/bitcoind.cpp new file mode 100644 index 0000000000..1e17ce4d3c --- /dev/null +++ b/src/init/bitcoind.cpp @@ -0,0 +1,29 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/init.h> +#include <node/context.h> + +#include <memory> + +namespace init { +namespace { +class BitcoindInit : public interfaces::Init +{ +public: + BitcoindInit(NodeContext& node) : m_node(node) + { + m_node.init = this; + } + NodeContext& m_node; +}; +} // namespace +} // namespace init + +namespace interfaces { +std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status) +{ + return std::make_unique<init::BitcoindInit>(node); +} +} // namespace interfaces diff --git a/src/init/common.cpp b/src/init/common.cpp new file mode 100644 index 0000000000..79e0c9da78 --- /dev/null +++ b/src/init/common.cpp @@ -0,0 +1,167 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <clientversion.h> +#include <compat/sanity.h> +#include <crypto/sha256.h> +#include <key.h> +#include <logging.h> +#include <node/ui_interface.h> +#include <pubkey.h> +#include <random.h> +#include <util/system.h> +#include <util/time.h> +#include <util/translation.h> + +#include <memory> + +static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; + +namespace init { +void SetGlobals() +{ + std::string sha256_algo = SHA256AutoDetect(); + LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); + RandomInit(); + ECC_Start(); + globalVerifyHandle.reset(new ECCVerifyHandle()); +} + +void UnsetGlobals() +{ + globalVerifyHandle.reset(); + ECC_Stop(); +} + +bool SanityChecks() +{ + if (!ECC_InitSanityCheck()) { + return InitError(Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")); + } + + if (!glibcxx_sanity_test()) + return false; + + if (!Random_SanityCheck()) { + return InitError(Untranslated("OS cryptographic RNG sanity check failure. Aborting.")); + } + + if (!ChronoSanityCheck()) { + return InitError(Untranslated("Clock epoch mismatch. Aborting.")); + } + + return true; +} + +void AddLoggingArgs(ArgsManager& argsman) +{ + argsman.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " + "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ". This option can be specified multiple times to output multiple categories.", + ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except the specified category. This option can be specified multiple times to exclude multiple categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +#ifdef HAVE_THREAD_LOCAL + argsman.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (only available on platforms supporting thread_local) (default: %u)", DEFAULT_LOGTHREADNAMES), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +#else + argsman.AddHiddenArgs({"-logthreadnames"}); +#endif + argsman.AddArg("-logsourcelocations", strprintf("Prepend debug output with name of the originating source location (source file, line number and function name) (default: %u)", DEFAULT_LOGSOURCELOCATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -daemon. To disable logging to file, set -nodebuglogfile)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-shrinkdebugfile", "Shrink debug.log file on client startup (default: 1 when no -debug)", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +} + +void SetLoggingOptions(const ArgsManager& args) +{ + LogInstance().m_print_to_file = !args.IsArgNegated("-debuglogfile"); + LogInstance().m_file_path = AbsPathForConfigVal(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); + LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", !args.GetBoolArg("-daemon", false)); + LogInstance().m_log_timestamps = args.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); + LogInstance().m_log_time_micros = args.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); +#ifdef HAVE_THREAD_LOCAL + LogInstance().m_log_threadnames = args.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); +#endif + LogInstance().m_log_sourcelocations = args.GetBoolArg("-logsourcelocations", DEFAULT_LOGSOURCELOCATIONS); + + fLogIPs = args.GetBoolArg("-logips", DEFAULT_LOGIPS); +} + +void SetLoggingCategories(const ArgsManager& args) +{ + if (args.IsArgSet("-debug")) { + // Special-case: if -debug=0/-nodebug is set, turn off debugging messages + const std::vector<std::string> categories = args.GetArgs("-debug"); + + if (std::none_of(categories.begin(), categories.end(), + [](std::string cat){return cat == "0" || cat == "none";})) { + for (const auto& cat : categories) { + if (!LogInstance().EnableCategory(cat)) { + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)); + } + } + } + } + + // Now remove the logging categories which were explicitly excluded + for (const std::string& cat : args.GetArgs("-debugexclude")) { + if (!LogInstance().DisableCategory(cat)) { + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); + } + } +} + +bool StartLogging(const ArgsManager& args) +{ + if (LogInstance().m_print_to_file) { + if (args.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { + // Do this first since it both loads a bunch of debug.log into memory, + // and because this needs to happen before any other debug.log printing + LogInstance().ShrinkDebugFile(); + } + } + if (!LogInstance().StartLogging()) { + return InitError(strprintf(Untranslated("Could not open debug log file %s"), + LogInstance().m_file_path.string())); + } + + if (!LogInstance().m_log_timestamps) + LogPrintf("Startup time: %s\n", FormatISO8601DateTime(GetTime())); + LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); + LogPrintf("Using data directory %s\n", GetDataDir().string()); + + // Only log conf file usage message if conf file actually exists. + fs::path config_file_path = GetConfigFile(args.GetArg("-conf", BITCOIN_CONF_FILENAME)); + if (fs::exists(config_file_path)) { + LogPrintf("Config file: %s\n", config_file_path.string()); + } else if (args.IsArgSet("-conf")) { + // Warn if no conf file exists at path provided by user + InitWarning(strprintf(_("The specified config file %s does not exist"), config_file_path.string())); + } else { + // Not categorizing as "Warning" because it's the default behavior + LogPrintf("Config file: %s (not found, skipping)\n", config_file_path.string()); + } + + // Log the config arguments to debug.log + args.LogArgs(); + + return true; +} + +void LogPackageVersion() +{ + std::string version_string = FormatFullVersion(); +#ifdef DEBUG + version_string += " (debug build)"; +#else + version_string += " (release build)"; +#endif + LogPrintf(PACKAGE_NAME " version %s\n", version_string); +} +} // namespace init diff --git a/src/init/common.h b/src/init/common.h new file mode 100644 index 0000000000..fc4bc1b280 --- /dev/null +++ b/src/init/common.h @@ -0,0 +1,28 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +//! @file +//! @brief Common init functions shared by bitcoin-node, bitcoin-wallet, etc. + +#ifndef BITCOIN_INIT_COMMON_H +#define BITCOIN_INIT_COMMON_H + +class ArgsManager; + +namespace init { +void SetGlobals(); +void UnsetGlobals(); +/** + * Ensure a usable environment with all + * necessary library support. + */ +bool SanityChecks(); +void AddLoggingArgs(ArgsManager& args); +void SetLoggingOptions(const ArgsManager& args); +void SetLoggingCategories(const ArgsManager& args); +bool StartLogging(const ArgsManager& args); +void LogPackageVersion(); +} // namespace init + +#endif // BITCOIN_INIT_COMMON_H diff --git a/src/interfaces/README.md b/src/interfaces/README.md index f77d172153..97167d5298 100644 --- a/src/interfaces/README.md +++ b/src/interfaces/README.md @@ -12,6 +12,8 @@ The following interfaces are defined here: * [`Handler`](handler.h) — returned by `handleEvent` methods on interfaces above and used to manage lifetimes of event handlers. -* [`Init`](init.h) — used by multiprocess code to access interfaces above on startup. Added in [#10102](https://github.com/bitcoin/bitcoin/pull/10102). +* [`Init`](init.h) — used by multiprocess code to access interfaces above on startup. Added in [#19160](https://github.com/bitcoin/bitcoin/pull/19160). -The interfaces above define boundaries between major components of bitcoin code (node, wallet, and gui), making it possible for them to run in different processes, and be tested, developed, and understood independently. These interfaces are not currently designed to be stable or to be used externally. +* [`Ipc`](ipc.h) — used by multiprocess code to access `Init` interface across processes. Added in [#19160](https://github.com/bitcoin/bitcoin/pull/19160). + +The interfaces above define boundaries between major components of bitcoin code (node, wallet, and gui), making it possible for them to run in [different processes](../../doc/multiprocess.md), and be tested, developed, and understood independently. These interfaces are not currently designed to be stable or to be used externally. diff --git a/src/interfaces/echo.cpp b/src/interfaces/echo.cpp new file mode 100644 index 0000000000..9bbb42217b --- /dev/null +++ b/src/interfaces/echo.cpp @@ -0,0 +1,18 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/echo.h> + +#include <memory> + +namespace interfaces { +namespace { +class EchoImpl : public Echo +{ +public: + std::string echo(const std::string& echo) override { return echo; } +}; +} // namespace +std::unique_ptr<Echo> MakeEcho() { return std::make_unique<EchoImpl>(); } +} // namespace interfaces diff --git a/src/interfaces/echo.h b/src/interfaces/echo.h new file mode 100644 index 0000000000..5578d9d9e6 --- /dev/null +++ b/src/interfaces/echo.h @@ -0,0 +1,26 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INTERFACES_ECHO_H +#define BITCOIN_INTERFACES_ECHO_H + +#include <memory> +#include <string> + +namespace interfaces { +//! Simple string echoing interface for testing. +class Echo +{ +public: + virtual ~Echo() {} + + //! Echo provided string. + virtual std::string echo(const std::string& echo) = 0; +}; + +//! Return implementation of Echo interface. +std::unique_ptr<Echo> MakeEcho(); +} // namespace interfaces + +#endif // BITCOIN_INTERFACES_ECHO_H diff --git a/src/interfaces/init.cpp b/src/interfaces/init.cpp new file mode 100644 index 0000000000..a3c949e616 --- /dev/null +++ b/src/interfaces/init.cpp @@ -0,0 +1,17 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/chain.h> +#include <interfaces/echo.h> +#include <interfaces/init.h> +#include <interfaces/node.h> +#include <interfaces/wallet.h> + +namespace interfaces { +std::unique_ptr<Node> Init::makeNode() { return {}; } +std::unique_ptr<Chain> Init::makeChain() { return {}; } +std::unique_ptr<WalletClient> Init::makeWalletClient(Chain& chain) { return {}; } +std::unique_ptr<Echo> Init::makeEcho() { return {}; } +Ipc* Init::ipc() { return nullptr; } +} // namespace interfaces diff --git a/src/interfaces/init.h b/src/interfaces/init.h new file mode 100644 index 0000000000..2a38054a17 --- /dev/null +++ b/src/interfaces/init.h @@ -0,0 +1,52 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INTERFACES_INIT_H +#define BITCOIN_INTERFACES_INIT_H + +#include <memory> + +struct NodeContext; + +namespace interfaces { +class Chain; +class Echo; +class Ipc; +class Node; +class WalletClient; + +//! Initial interface created when a process is first started, and used to give +//! and get access to other interfaces (Node, Chain, Wallet, etc). +//! +//! There is a different Init interface implementation for each process +//! (bitcoin-gui, bitcoin-node, bitcoin-wallet, bitcoind, bitcoin-qt) and each +//! implementation can implement the make methods for interfaces it supports. +//! The default make methods all return null. +class Init +{ +public: + virtual ~Init() = default; + virtual std::unique_ptr<Node> makeNode(); + virtual std::unique_ptr<Chain> makeChain(); + virtual std::unique_ptr<WalletClient> makeWalletClient(Chain& chain); + virtual std::unique_ptr<Echo> makeEcho(); + virtual Ipc* ipc(); +}; + +//! Return implementation of Init interface for the node process. If the argv +//! indicates that this is a child process spawned to handle requests from a +//! parent process, this blocks and handles requests, then returns null and a +//! status code to exit with. If this returns non-null, the caller can start up +//! normally and use the Init object to spawn and connect to other processes +//! while it is running. +std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status); + +//! Return implementation of Init interface for the wallet process. +std::unique_ptr<Init> MakeWalletInit(int argc, char* argv[], int& exit_status); + +//! Return implementation of Init interface for the gui process. +std::unique_ptr<Init> MakeGuiInit(int argc, char* argv[]); +} // namespace interfaces + +#endif // BITCOIN_INTERFACES_INIT_H diff --git a/src/interfaces/ipc.h b/src/interfaces/ipc.h new file mode 100644 index 0000000000..e9e6c78053 --- /dev/null +++ b/src/interfaces/ipc.h @@ -0,0 +1,71 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INTERFACES_IPC_H +#define BITCOIN_INTERFACES_IPC_H + +#include <functional> +#include <memory> +#include <typeindex> + +namespace interfaces { +class Init; + +//! Interface providing access to interprocess-communication (IPC) +//! functionality. The IPC implementation is responsible for establishing +//! connections between a controlling process and a process being controlled. +//! When a connection is established, the process being controlled returns an +//! interfaces::Init pointer to the controlling process, which the controlling +//! process can use to get access to other interfaces and functionality. +//! +//! When spawning a new process, the steps are: +//! +//! 1. The controlling process calls interfaces::Ipc::spawnProcess(), which +//! calls ipc::Process::spawn(), which spawns a new process and returns a +//! socketpair file descriptor for communicating with it. +//! interfaces::Ipc::spawnProcess() then calls ipc::Protocol::connect() +//! passing the socketpair descriptor, which returns a local proxy +//! interfaces::Init implementation calling remote interfaces::Init methods. +//! 2. The spawned process calls interfaces::Ipc::startSpawnProcess(), which +//! calls ipc::Process::checkSpawned() to read command line arguments and +//! determine whether it is a spawned process and what socketpair file +//! descriptor it should use. It then calls ipc::Protocol::serve() to handle +//! incoming requests from the socketpair and invoke interfaces::Init +//! interface methods, and exit when the socket is closed. +//! 3. The controlling process calls local proxy interfaces::Init object methods +//! to make other proxy objects calling other remote interfaces. It can also +//! destroy the initial interfaces::Init object to close the connection and +//! shut down the spawned process. +class Ipc +{ +public: + virtual ~Ipc() = default; + + //! Spawn a child process returning pointer to its Init interface. + virtual std::unique_ptr<Init> spawnProcess(const char* exe_name) = 0; + + //! If this is a spawned process, block and handle requests from the parent + //! process by forwarding them to this process's Init interface, then return + //! true. If this is not a spawned child process, return false. + virtual bool startSpawnedProcess(int argc, char* argv[], int& exit_status) = 0; + + //! Add cleanup callback to remote interface that will run when the + //! interface is deleted. + template<typename Interface> + void addCleanup(Interface& iface, std::function<void()> cleanup) + { + addCleanup(typeid(Interface), &iface, std::move(cleanup)); + } + +protected: + //! Internal implementation of public addCleanup method (above) as a + //! type-erased virtual function, since template functions can't be virtual. + virtual void addCleanup(std::type_index type, void* iface, std::function<void()> cleanup) = 0; +}; + +//! Return implementation of Ipc interface. +std::unique_ptr<Ipc> MakeIpc(const char* exe_name, const char* process_argv0, Init& init); +} // namespace interfaces + +#endif // BITCOIN_INTERFACES_IPC_H diff --git a/src/ipc/capnp/.gitignore b/src/ipc/capnp/.gitignore new file mode 100644 index 0000000000..036df1430c --- /dev/null +++ b/src/ipc/capnp/.gitignore @@ -0,0 +1,2 @@ +# capnp generated files +*.capnp.* diff --git a/src/ipc/capnp/echo.capnp b/src/ipc/capnp/echo.capnp new file mode 100644 index 0000000000..df36ee0de3 --- /dev/null +++ b/src/ipc/capnp/echo.capnp @@ -0,0 +1,17 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0x888b4f7f51e691f7; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("ipc::capnp::messages"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("interfaces/echo.h"); +$Proxy.include("ipc/capnp/echo.capnp.h"); + +interface Echo $Proxy.wrap("interfaces::Echo") { + destroy @0 (context :Proxy.Context) -> (); + echo @1 (context :Proxy.Context, echo: Text) -> (result :Text); +} diff --git a/src/ipc/capnp/init-types.h b/src/ipc/capnp/init-types.h new file mode 100644 index 0000000000..42031441b5 --- /dev/null +++ b/src/ipc/capnp/init-types.h @@ -0,0 +1,10 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_INIT_TYPES_H +#define BITCOIN_IPC_CAPNP_INIT_TYPES_H + +#include <ipc/capnp/echo.capnp.proxy-types.h> + +#endif // BITCOIN_IPC_CAPNP_INIT_TYPES_H diff --git a/src/ipc/capnp/init.capnp b/src/ipc/capnp/init.capnp new file mode 100644 index 0000000000..e6d358c665 --- /dev/null +++ b/src/ipc/capnp/init.capnp @@ -0,0 +1,20 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xf2c5cfa319406aa6; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("ipc::capnp::messages"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("interfaces/echo.h"); +$Proxy.include("interfaces/init.h"); +$Proxy.includeTypes("ipc/capnp/init-types.h"); + +using Echo = import "echo.capnp"; + +interface Init $Proxy.wrap("interfaces::Init") { + construct @0 (threadMap: Proxy.ThreadMap) -> (threadMap :Proxy.ThreadMap); + makeEcho @1 (context :Proxy.Context) -> (result :Echo.Echo); +} diff --git a/src/ipc/capnp/protocol.cpp b/src/ipc/capnp/protocol.cpp new file mode 100644 index 0000000000..74c66c899a --- /dev/null +++ b/src/ipc/capnp/protocol.cpp @@ -0,0 +1,90 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/init.h> +#include <ipc/capnp/init.capnp.h> +#include <ipc/capnp/init.capnp.proxy.h> +#include <ipc/capnp/protocol.h> +#include <ipc/exception.h> +#include <ipc/protocol.h> +#include <kj/async.h> +#include <logging.h> +#include <mp/proxy-io.h> +#include <mp/proxy-types.h> +#include <mp/util.h> +#include <util/threadnames.h> + +#include <assert.h> +#include <errno.h> +#include <future> +#include <memory> +#include <mutex> +#include <optional> +#include <string> +#include <thread> + +namespace ipc { +namespace capnp { +namespace { +void IpcLogFn(bool raise, std::string message) +{ + LogPrint(BCLog::IPC, "%s\n", message); + if (raise) throw Exception(message); +} + +class CapnpProtocol : public Protocol +{ +public: + ~CapnpProtocol() noexcept(true) + { + if (m_loop) { + std::unique_lock<std::mutex> lock(m_loop->m_mutex); + m_loop->removeClient(lock); + } + if (m_loop_thread.joinable()) m_loop_thread.join(); + assert(!m_loop); + }; + std::unique_ptr<interfaces::Init> connect(int fd, const char* exe_name) override + { + startLoop(exe_name); + return mp::ConnectStream<messages::Init>(*m_loop, fd); + } + void serve(int fd, const char* exe_name, interfaces::Init& init) override + { + assert(!m_loop); + mp::g_thread_context.thread_name = mp::ThreadName(exe_name); + m_loop.emplace(exe_name, &IpcLogFn, nullptr); + mp::ServeStream<messages::Init>(*m_loop, fd, init); + m_loop->loop(); + m_loop.reset(); + } + void addCleanup(std::type_index type, void* iface, std::function<void()> cleanup) override + { + mp::ProxyTypeRegister::types().at(type)(iface).cleanup.emplace_back(std::move(cleanup)); + } + void startLoop(const char* exe_name) + { + if (m_loop) return; + std::promise<void> promise; + m_loop_thread = std::thread([&] { + util::ThreadRename("capnp-loop"); + m_loop.emplace(exe_name, &IpcLogFn, nullptr); + { + std::unique_lock<std::mutex> lock(m_loop->m_mutex); + m_loop->addClient(lock); + } + promise.set_value(); + m_loop->loop(); + m_loop.reset(); + }); + promise.get_future().wait(); + } + std::thread m_loop_thread; + std::optional<mp::EventLoop> m_loop; +}; +} // namespace + +std::unique_ptr<Protocol> MakeCapnpProtocol() { return std::make_unique<CapnpProtocol>(); } +} // namespace capnp +} // namespace ipc diff --git a/src/ipc/capnp/protocol.h b/src/ipc/capnp/protocol.h new file mode 100644 index 0000000000..eb057949d2 --- /dev/null +++ b/src/ipc/capnp/protocol.h @@ -0,0 +1,17 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_PROTOCOL_H +#define BITCOIN_IPC_CAPNP_PROTOCOL_H + +#include <memory> + +namespace ipc { +class Protocol; +namespace capnp { +std::unique_ptr<Protocol> MakeCapnpProtocol(); +} // namespace capnp +} // namespace ipc + +#endif // BITCOIN_IPC_CAPNP_PROTOCOL_H diff --git a/src/ipc/exception.h b/src/ipc/exception.h new file mode 100644 index 0000000000..53dee8124a --- /dev/null +++ b/src/ipc/exception.h @@ -0,0 +1,20 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_EXCEPTION_H +#define BITCOIN_IPC_EXCEPTION_H + +#include <stdexcept> + +namespace ipc { +//! Exception class thrown when a call to remote method fails due to an IPC +//! error, like a socket getting disconnected. +class Exception : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; +} // namespace ipc + +#endif // BITCOIN_IPC_EXCEPTION_H diff --git a/src/ipc/interfaces.cpp b/src/ipc/interfaces.cpp new file mode 100644 index 0000000000..ad4b78ed81 --- /dev/null +++ b/src/ipc/interfaces.cpp @@ -0,0 +1,77 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <fs.h> +#include <interfaces/init.h> +#include <interfaces/ipc.h> +#include <ipc/capnp/protocol.h> +#include <ipc/process.h> +#include <ipc/protocol.h> +#include <logging.h> +#include <tinyformat.h> +#include <util/system.h> + +#include <functional> +#include <memory> +#include <stdexcept> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <string> +#include <unistd.h> +#include <utility> +#include <vector> + +namespace ipc { +namespace { +class IpcImpl : public interfaces::Ipc +{ +public: + IpcImpl(const char* exe_name, const char* process_argv0, interfaces::Init& init) + : m_exe_name(exe_name), m_process_argv0(process_argv0), m_init(init), + m_protocol(ipc::capnp::MakeCapnpProtocol()), m_process(ipc::MakeProcess()) + { + } + std::unique_ptr<interfaces::Init> spawnProcess(const char* new_exe_name) override + { + int pid; + int fd = m_process->spawn(new_exe_name, m_process_argv0, pid); + LogPrint(::BCLog::IPC, "Process %s pid %i launched\n", new_exe_name, pid); + auto init = m_protocol->connect(fd, m_exe_name); + Ipc::addCleanup(*init, [this, new_exe_name, pid] { + int status = m_process->waitSpawned(pid); + LogPrint(::BCLog::IPC, "Process %s pid %i exited with status %i\n", new_exe_name, pid, status); + }); + return init; + } + bool startSpawnedProcess(int argc, char* argv[], int& exit_status) override + { + exit_status = EXIT_FAILURE; + int32_t fd = -1; + if (!m_process->checkSpawned(argc, argv, fd)) { + return false; + } + m_protocol->serve(fd, m_exe_name, m_init); + exit_status = EXIT_SUCCESS; + return true; + } + void addCleanup(std::type_index type, void* iface, std::function<void()> cleanup) override + { + m_protocol->addCleanup(type, iface, std::move(cleanup)); + } + const char* m_exe_name; + const char* m_process_argv0; + interfaces::Init& m_init; + std::unique_ptr<Protocol> m_protocol; + std::unique_ptr<Process> m_process; +}; +} // namespace +} // namespace ipc + +namespace interfaces { +std::unique_ptr<Ipc> MakeIpc(const char* exe_name, const char* process_argv0, Init& init) +{ + return std::make_unique<ipc::IpcImpl>(exe_name, process_argv0, init); +} +} // namespace interfaces diff --git a/src/ipc/process.cpp b/src/ipc/process.cpp new file mode 100644 index 0000000000..43ed1f1bae --- /dev/null +++ b/src/ipc/process.cpp @@ -0,0 +1,61 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <fs.h> +#include <ipc/process.h> +#include <ipc/protocol.h> +#include <mp/util.h> +#include <tinyformat.h> +#include <util/strencodings.h> + +#include <cstdint> +#include <exception> +#include <iostream> +#include <stdexcept> +#include <stdlib.h> +#include <string.h> +#include <system_error> +#include <unistd.h> +#include <utility> +#include <vector> + +namespace ipc { +namespace { +class ProcessImpl : public Process +{ +public: + int spawn(const std::string& new_exe_name, const fs::path& argv0_path, int& pid) override + { + return mp::SpawnProcess(pid, [&](int fd) { + fs::path path = argv0_path; + path.remove_filename(); + path.append(new_exe_name); + return std::vector<std::string>{path.string(), "-ipcfd", strprintf("%i", fd)}; + }); + } + int waitSpawned(int pid) override { return mp::WaitProcess(pid); } + bool checkSpawned(int argc, char* argv[], int& fd) override + { + // If this process was not started with a single -ipcfd argument, it is + // not a process spawned by the spawn() call above, so return false and + // do not try to serve requests. + if (argc != 3 || strcmp(argv[1], "-ipcfd") != 0) { + return false; + } + // If a single -ipcfd argument was provided, return true and get the + // file descriptor so Protocol::serve() can be called to handle + // requests from the parent process. The -ipcfd argument is not valid + // in combination with other arguments because the parent process + // should be able to control the child process through the IPC protocol + // without passing information out of band. + if (!ParseInt32(argv[2], &fd)) { + throw std::runtime_error(strprintf("Invalid -ipcfd number '%s'", argv[2])); + } + return true; + } +}; +} // namespace + +std::unique_ptr<Process> MakeProcess() { return std::make_unique<ProcessImpl>(); } +} // namespace ipc diff --git a/src/ipc/process.h b/src/ipc/process.h new file mode 100644 index 0000000000..4bb2930d9c --- /dev/null +++ b/src/ipc/process.h @@ -0,0 +1,42 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_PROCESS_H +#define BITCOIN_IPC_PROCESS_H + +#include <memory> +#include <string> + +namespace ipc { +class Protocol; + +//! IPC process interface for spawning bitcoin processes and serving requests +//! in processes that have been spawned. +//! +//! There will be different implementations of this interface depending on the +//! platform (e.g. unix, windows). +class Process +{ +public: + virtual ~Process() = default; + + //! Spawn process and return socket file descriptor for communicating with + //! it. + virtual int spawn(const std::string& new_exe_name, const fs::path& argv0_path, int& pid) = 0; + + //! Wait for spawned process to exit and return its exit code. + virtual int waitSpawned(int pid) = 0; + + //! Parse command line and determine if current process is a spawned child + //! process. If so, return true and a file descriptor for communicating + //! with the parent process. + virtual bool checkSpawned(int argc, char* argv[], int& fd) = 0; +}; + +//! Constructor for Process interface. Implementation will vary depending on +//! the platform (unix or windows). +std::unique_ptr<Process> MakeProcess(); +} // namespace ipc + +#endif // BITCOIN_IPC_PROCESS_H diff --git a/src/ipc/protocol.h b/src/ipc/protocol.h new file mode 100644 index 0000000000..af955b0007 --- /dev/null +++ b/src/ipc/protocol.h @@ -0,0 +1,39 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_PROTOCOL_H +#define BITCOIN_IPC_PROTOCOL_H + +#include <interfaces/init.h> + +#include <functional> +#include <memory> +#include <typeindex> + +namespace ipc { +//! IPC protocol interface for calling IPC methods over sockets. +//! +//! There may be different implementations of this interface for different IPC +//! protocols (e.g. Cap'n Proto, gRPC, JSON-RPC, or custom protocols). +class Protocol +{ +public: + virtual ~Protocol() = default; + + //! Return Init interface that forwards requests over given socket descriptor. + //! Socket communication is handled on a background thread. + virtual std::unique_ptr<interfaces::Init> connect(int fd, const char* exe_name) = 0; + + //! Handle requests on provided socket descriptor, forwarding them to the + //! provided Init interface. Socket communication is handled on the + //! current thread, and this call blocks until the socket is closed. + virtual void serve(int fd, const char* exe_name, interfaces::Init& init) = 0; + + //! Add cleanup callback to interface that will run when the interface is + //! deleted. + virtual void addCleanup(std::type_index type, void* iface, std::function<void()> cleanup) = 0; +}; +} // namespace ipc + +#endif // BITCOIN_IPC_PROTOCOL_H diff --git a/src/key.cpp b/src/key.cpp index 1e59b301cb..5666adebb8 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -293,7 +293,7 @@ bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const bool CExtKey::Derive(CExtKey &out, unsigned int _nChild) const { out.nDepth = nDepth + 1; CKeyID id = key.GetPubKey().GetID(); - memcpy(&out.vchFingerprint[0], &id, 4); + memcpy(out.vchFingerprint, &id, 4); out.nChild = _nChild; return key.Derive(out.key, out.chaincode, _nChild, chaincode); } @@ -312,7 +312,7 @@ void CExtKey::SetSeed(const unsigned char *seed, unsigned int nSeedLen) { CExtPubKey CExtKey::Neuter() const { CExtPubKey ret; ret.nDepth = nDepth; - memcpy(&ret.vchFingerprint[0], &vchFingerprint[0], 4); + memcpy(ret.vchFingerprint, vchFingerprint, 4); ret.nChild = nChild; ret.pubkey = key.GetPubKey(); ret.chaincode = chaincode; @@ -151,7 +151,7 @@ struct CExtKey { friend bool operator==(const CExtKey& a, const CExtKey& b) { return a.nDepth == b.nDepth && - memcmp(&a.vchFingerprint[0], &b.vchFingerprint[0], sizeof(vchFingerprint)) == 0 && + memcmp(a.vchFingerprint, b.vchFingerprint, sizeof(vchFingerprint)) == 0 && a.nChild == b.nChild && a.chaincode == b.chaincode && a.key == b.key; diff --git a/src/logging.cpp b/src/logging.cpp index 866213786e..e5187fd596 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -157,6 +157,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::LEVELDB, "leveldb"}, {BCLog::VALIDATION, "validation"}, {BCLog::I2P, "i2p"}, + {BCLog::IPC, "ipc"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; diff --git a/src/logging.h b/src/logging.h index 436f0cd12e..d04bc99268 100644 --- a/src/logging.h +++ b/src/logging.h @@ -58,6 +58,7 @@ namespace BCLog { LEVELDB = (1 << 20), VALIDATION = (1 << 21), I2P = (1 << 22), + IPC = (1 << 23), ALL = ~(uint32_t)0, }; diff --git a/src/net.cpp b/src/net.cpp index 5aa267f0d7..ce38c140ce 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -681,19 +681,19 @@ int V1TransportDeserializer::readHeader(Span<const uint8_t> msg_bytes) hdrbuf >> hdr; } catch (const std::exception&) { - LogPrint(BCLog::NET, "HEADER ERROR - UNABLE TO DESERIALIZE, peer=%d\n", m_node_id); + LogPrint(BCLog::NET, "Header error: Unable to deserialize, peer=%d\n", m_node_id); return -1; } // Check start string, network magic if (memcmp(hdr.pchMessageStart, m_chain_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrint(BCLog::NET, "HEADER ERROR - MESSAGESTART (%s, %u bytes), received %s, peer=%d\n", hdr.GetCommand(), hdr.nMessageSize, HexStr(hdr.pchMessageStart), m_node_id); + LogPrint(BCLog::NET, "Header error: Wrong MessageStart %s received, peer=%d\n", HexStr(hdr.pchMessageStart), m_node_id); return -1; } // reject messages larger than MAX_SIZE or MAX_PROTOCOL_MESSAGE_LENGTH if (hdr.nMessageSize > MAX_SIZE || hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { - LogPrint(BCLog::NET, "HEADER ERROR - SIZE (%s, %u bytes), peer=%d\n", hdr.GetCommand(), hdr.nMessageSize, m_node_id); + LogPrint(BCLog::NET, "Header error: Size too large (%s, %u bytes), peer=%d\n", SanitizeString(hdr.GetCommand()), hdr.nMessageSize, m_node_id); return -1; } @@ -746,7 +746,7 @@ std::optional<CNetMessage> V1TransportDeserializer::GetMessage(const std::chrono // Check checksum and header command string if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { - LogPrint(BCLog::NET, "CHECKSUM ERROR (%s, %u bytes), expected %s was %s, peer=%d\n", + LogPrint(BCLog::NET, "Header error: Wrong checksum (%s, %u bytes), expected %s was %s, peer=%d\n", SanitizeString(msg->m_command), msg->m_message_size, HexStr(Span<uint8_t>(hash.begin(), hash.begin() + CMessageHeader::CHECKSUM_SIZE)), HexStr(hdr.pchChecksum), @@ -754,8 +754,8 @@ std::optional<CNetMessage> V1TransportDeserializer::GetMessage(const std::chrono out_err_raw_size = msg->m_raw_message_size; msg = std::nullopt; } else if (!hdr.IsCommandValid()) { - LogPrint(BCLog::NET, "HEADER ERROR - COMMAND (%s, %u bytes), peer=%d\n", - hdr.GetCommand(), msg->m_message_size, m_node_id); + LogPrint(BCLog::NET, "Header error: Invalid message type (%s, %u bytes), peer=%d\n", + SanitizeString(hdr.GetCommand()), msg->m_message_size, m_node_id); out_err_raw_size = msg->m_raw_message_size; msg.reset(); } @@ -1224,19 +1224,10 @@ void CConnman::DisconnectNodes() std::list<CNode*> vNodesDisconnectedCopy = vNodesDisconnected; for (CNode* pnode : vNodesDisconnectedCopy) { - // wait until threads are done using it + // Destroy the object only after other threads have stopped using it. if (pnode->GetRefCount() <= 0) { - bool fDelete = false; - { - TRY_LOCK(pnode->cs_vSend, lockSend); - if (lockSend) { - fDelete = true; - } - } - if (fDelete) { - vNodesDisconnected.remove(pnode); - DeleteNode(pnode); - } + vNodesDisconnected.remove(pnode); + DeleteNode(pnode); } } } @@ -2635,23 +2626,26 @@ void CConnman::StopNodes() } } - // Close sockets - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) + // Delete peer connections. + std::vector<CNode*> nodes; + WITH_LOCK(cs_vNodes, nodes.swap(vNodes)); + for (CNode* pnode : nodes) { pnode->CloseSocketDisconnect(); - for (ListenSocket& hListenSocket : vhListenSocket) - if (hListenSocket.socket != INVALID_SOCKET) - if (!CloseSocket(hListenSocket.socket)) - LogPrintf("CloseSocket(hListenSocket) failed with error %s\n", NetworkErrorString(WSAGetLastError())); - - // clean up some globals (to help leak detection) - for (CNode* pnode : vNodes) { DeleteNode(pnode); } + + // Close listening sockets. + for (ListenSocket& hListenSocket : vhListenSocket) { + if (hListenSocket.socket != INVALID_SOCKET) { + if (!CloseSocket(hListenSocket.socket)) { + LogPrintf("CloseSocket(hListenSocket) failed with error %s\n", NetworkErrorString(WSAGetLastError())); + } + } + } + for (CNode* pnode : vNodesDisconnected) { DeleteNode(pnode); } - vNodes.clear(); vNodesDisconnected.clear(); vhListenSocket.clear(); semOutbound.reset(); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 2201caf7d2..27ad9eefb5 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -25,6 +25,7 @@ #include <reverse_iterator.h> #include <scheduler.h> #include <streams.h> +#include <sync.h> #include <tinyformat.h> #include <txmempool.h> #include <txorphanage.h> @@ -256,6 +257,9 @@ public: const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) override; private: + void _RelayTransaction(const uint256& txid, const uint256& wtxid) + EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ void ConsiderEviction(CNode& pto, int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -1015,7 +1019,7 @@ void PeerManagerImpl::ReattemptInitialBroadcast(CScheduler& scheduler) if (tx != nullptr) { LOCK(cs_main); - RelayTransaction(txid, tx->GetWitnessHash()); + _RelayTransaction(txid, tx->GetWitnessHash()); } else { m_mempool.RemoveUnbroadcastTx(txid, true); } @@ -1512,6 +1516,11 @@ void PeerManagerImpl::SendPings() void PeerManagerImpl::RelayTransaction(const uint256& txid, const uint256& wtxid) { + WITH_LOCK(cs_main, _RelayTransaction(txid, wtxid);); +} + +void PeerManagerImpl::_RelayTransaction(const uint256& txid, const uint256& wtxid) +{ m_connman.ForEachNode([&txid, &wtxid](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); @@ -2087,7 +2096,7 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanHash, porphanTx->GetWitnessHash()); + _RelayTransaction(orphanHash, porphanTx->GetWitnessHash()); m_orphanage.AddChildrenToWorkSet(*porphanTx, orphan_work_set); m_orphanage.EraseTx(orphanHash); for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { @@ -3055,7 +3064,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); - RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); + _RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); } } return; @@ -3070,7 +3079,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // requests for it. m_txrequest.ForgetTxHash(tx.GetHash()); m_txrequest.ForgetTxHash(tx.GetWitnessHash()); - RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); + _RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); m_orphanage.AddChildrenToWorkSet(tx, peer->m_orphan_work_set); pfrom.nLastTXTime = GetTime(); @@ -4449,8 +4458,9 @@ bool PeerManagerImpl::SendMessages(CNode* pto) } } peer->m_blocks_for_inv_relay.clear(); + } - if (pto->m_tx_relay != nullptr) { + if (pto->m_tx_relay != nullptr) { LOCK(pto->m_tx_relay->cs_tx_inventory); // Check whether periodic sends should happen bool fSendTrickle = pto->HasPermission(PF_NOBAN); @@ -4578,7 +4588,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) } } } - } } if (!vInv.empty()) m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); diff --git a/src/net_processing.h b/src/net_processing.h index 67252acbb6..d5801aadd3 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -7,7 +7,6 @@ #define BITCOIN_NET_PROCESSING_H #include <net.h> -#include <sync.h> #include <validationinterface.h> class CAddrMan; @@ -15,8 +14,6 @@ class CChainParams; class CTxMemPool; class ChainstateManager; -extern RecursiveMutex cs_main; - /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; /** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ @@ -49,8 +46,7 @@ public: virtual bool IgnoresIncomingTxs() = 0; /** Relay transaction to all peers. */ - virtual void RelayTransaction(const uint256& txid, const uint256& wtxid) - EXCLUSIVE_LOCKS_REQUIRED(cs_main) = 0; + virtual void RelayTransaction(const uint256& txid, const uint256& wtxid) = 0; /** Send ping message to all peers */ virtual void SendPings() = 0; diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 69edc15c66..d56ae78e92 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -551,6 +551,11 @@ enum Network CNetAddr::GetNetwork() const return m_net; } +static std::string IPv4ToString(Span<const uint8_t> a) +{ + return strprintf("%u.%u.%u.%u", a[0], a[1], a[2], a[3]); +} + static std::string IPv6ToString(Span<const uint8_t> a) { assert(a.size() == ADDR_IPV6_SIZE); @@ -571,6 +576,7 @@ std::string CNetAddr::ToStringIP() const { switch (m_net) { case NET_IPV4: + return IPv4ToString(m_addr); case NET_IPV6: { CService serv(*this, 0); struct sockaddr_storage sockaddr; @@ -581,9 +587,6 @@ std::string CNetAddr::ToStringIP() const sizeof(name), nullptr, 0, NI_NUMERICHOST)) return std::string(name); } - if (m_net == NET_IPV4) { - return strprintf("%u.%u.%u.%u", m_addr[0], m_addr[1], m_addr[2], m_addr[3]); - } return IPv6ToString(m_addr); } case NET_ONION: diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index daed6605e8..6c66c565ad 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -6,17 +6,310 @@ #include <chain.h> #include <chainparams.h> +#include <clientversion.h> +#include <consensus/validation.h> #include <flatfile.h> #include <fs.h> +#include <hash.h> #include <pow.h> #include <shutdown.h> #include <signet.h> #include <streams.h> +#include <undo.h> #include <util/system.h> #include <validation.h> -// From validation. TODO move here -bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false); +std::atomic_bool fImporting(false); +std::atomic_bool fReindex(false); +bool fHavePruned = false; +bool fPruneMode = false; +uint64_t nPruneTarget = 0; + +// TODO make namespace { +RecursiveMutex cs_LastBlockFile; +std::vector<CBlockFileInfo> vinfoBlockFile; +int nLastBlockFile = 0; +/** Global flag to indicate we should check to see if there are +* block/undo files that should be deleted. Set on startup +* or if we allocate more file space when we're in prune mode +*/ +bool fCheckForPruning = false; + +/** Dirty block index entries. */ +std::set<CBlockIndex*> setDirtyBlockIndex; + +/** Dirty block file entries. */ +std::set<int> setDirtyFileInfo; +// } // namespace + +static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); + +bool IsBlockPruned(const CBlockIndex* pblockindex) +{ + return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); +} + +// If we're using -prune with -reindex, then delete block files that will be ignored by the +// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile +// is missing, do the same here to delete any later block files after a gap. Also delete all +// rev files since they'll be rewritten by the reindex anyway. This ensures that vinfoBlockFile +// is in sync with what's actually on disk by the time we start downloading, so that pruning +// works correctly. +void CleanupBlockRevFiles() +{ + std::map<std::string, fs::path> mapBlockFiles; + + // Glob all blk?????.dat and rev?????.dat files from the blocks directory. + // Remove the rev files immediately and insert the blk file paths into an + // ordered map keyed by block file index. + LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); + fs::path blocksdir = gArgs.GetBlocksDirPath(); + for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { + if (fs::is_regular_file(*it) && + it->path().filename().string().length() == 12 && + it->path().filename().string().substr(8,4) == ".dat") + { + if (it->path().filename().string().substr(0, 3) == "blk") { + mapBlockFiles[it->path().filename().string().substr(3, 5)] = it->path(); + } else if (it->path().filename().string().substr(0, 3) == "rev") { + remove(it->path()); + } + } + } + + // Remove all block files that aren't part of a contiguous set starting at + // zero by walking the ordered map (keys are block file indices) by + // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) + // start removing block files. + int nContigCounter = 0; + for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) { + if (atoi(item.first) == nContigCounter) { + nContigCounter++; + continue; + } + remove(item.second); + } +} + +std::string CBlockFileInfo::ToString() const +{ + return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); +} + +CBlockFileInfo* GetBlockFileInfo(size_t n) +{ + LOCK(cs_LastBlockFile); + + return &vinfoBlockFile.at(n); +} + +static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +{ + // Open history file to append + CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) { + return error("%s: OpenUndoFile failed", __func__); + } + + // Write index header + unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); + fileout << messageStart << nSize; + + // Write undo data + long fileOutPos = ftell(fileout.Get()); + if (fileOutPos < 0) { + return error("%s: ftell failed", __func__); + } + pos.nPos = (unsigned int)fileOutPos; + fileout << blockundo; + + // calculate & write checksum + CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); + hasher << hashBlock; + hasher << blockundo; + fileout << hasher.GetHash(); + + return true; +} + +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) +{ + FlatFilePos pos = pindex->GetUndoPos(); + if (pos.IsNull()) { + return error("%s: no undo data available", __func__); + } + + // Open history file to read + CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) { + return error("%s: OpenUndoFile failed", __func__); + } + + // Read block + uint256 hashChecksum; + CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data + try { + verifier << pindex->pprev->GetBlockHash(); + verifier >> blockundo; + filein >> hashChecksum; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + + // Verify checksum + if (hashChecksum != verifier.GetHash()) { + return error("%s: Checksum mismatch", __func__); + } + + return true; +} + +static void FlushUndoFile(int block_file, bool finalize = false) +{ + FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize); + if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { + AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); + } +} + +void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false) +{ + LOCK(cs_LastBlockFile); + FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { + AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); + } + // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, + // e.g. during IBD or a sync after a node going offline + if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo); +} + +uint64_t CalculateCurrentUsage() +{ + LOCK(cs_LastBlockFile); + + uint64_t retval = 0; + for (const CBlockFileInfo& file : vinfoBlockFile) { + retval += file.nSize + file.nUndoSize; + } + return retval; +} + +void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) +{ + for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { + FlatFilePos pos(*it, 0); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); + LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + } +} + +static FlatFileSeq BlockFileSeq() +{ + return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); +} + +static FlatFileSeq UndoFileSeq() +{ + return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); +} + +FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) +{ + return BlockFileSeq().Open(pos, fReadOnly); +} + +/** Open an undo file (rev?????.dat) */ +static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) +{ + return UndoFileSeq().Open(pos, fReadOnly); +} + +fs::path GetBlockPosFilename(const FlatFilePos& pos) +{ + return BlockFileSeq().FileName(pos); +} + +bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false) +{ + LOCK(cs_LastBlockFile); + + unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; + if (vinfoBlockFile.size() <= nFile) { + vinfoBlockFile.resize(nFile + 1); + } + + bool finalize_undo = false; + if (!fKnown) { + while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { + // when the undo file is keeping up with the block file, we want to flush it explicitly + // when it is lagging behind (more blocks arrive than are being connected), we let the + // undo block write case handle it + assert(std::addressof(::ChainActive()) == std::addressof(active_chain)); + finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); + nFile++; + if (vinfoBlockFile.size() <= nFile) { + vinfoBlockFile.resize(nFile + 1); + } + } + pos.nFile = nFile; + pos.nPos = vinfoBlockFile[nFile].nSize; + } + + if ((int)nFile != nLastBlockFile) { + if (!fKnown) { + LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); + } + FlushBlockFile(!fKnown, finalize_undo); + nLastBlockFile = nFile; + } + + vinfoBlockFile[nFile].AddBlock(nHeight, nTime); + if (fKnown) { + vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); + } else { + vinfoBlockFile[nFile].nSize += nAddSize; + } + + if (!fKnown) { + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is too low!", _("Disk space is too low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; + } + } + + setDirtyFileInfo.insert(nFile); + return true; +} + +static bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize) +{ + pos.nFile = nFile; + + LOCK(cs_LastBlockFile); + + pos.nPos = vinfoBlockFile[nFile].nUndoSize; + vinfoBlockFile[nFile].nUndoSize += nAddSize; + setDirtyFileInfo.insert(nFile); + + bool out_of_space; + size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; + } + + return true; +} static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { @@ -41,6 +334,35 @@ static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessa return true; } +bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +{ + // Write undo information to disk + if (pindex->GetUndoPos().IsNull()) { + FlatFilePos _pos; + if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) { + return error("ConnectBlock(): FindUndoPos failed"); + } + if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) { + return AbortNode(state, "Failed to write undo data"); + } + // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) + // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height + // in the block file info as below; note that this does not catch the case where the undo writes are keeping up + // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in + // the FindBlockPos function + if (_pos.nFile < nLastBlockFile && static_cast<uint32_t>(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) { + FlushUndoFile(_pos.nFile, true); + } + + // update nUndoPos in block index + pindex->nUndoPos = _pos.nPos; + pindex->nStatus |= BLOCK_HAVE_UNDO; + setDirtyBlockIndex.insert(pindex); + } + + return true; +} + bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 3b546f0719..faf99dea81 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -12,7 +12,9 @@ #include <protocol.h> // For CMessageHeader::MessageStartChars class ArgsManager; +class BlockValidationState; class CBlock; +class CBlockFileInfo; class CBlockIndex; class CBlockUndo; class CChain; @@ -25,6 +27,44 @@ struct Params; static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; +/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ +static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ +static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** The maximum size of a blk?????.dat file (since 0.8) */ +static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB + +extern std::atomic_bool fImporting; +extern std::atomic_bool fReindex; +/** Pruning-related variables and constants */ +/** True if any block files have ever been pruned. */ +extern bool fHavePruned; +/** True if we're running in -prune mode. */ +extern bool fPruneMode; +/** Number of MiB of block files that we're trying to stay below. */ +extern uint64_t nPruneTarget; + +//! Check whether the block associated with this index entry is pruned or not. +bool IsBlockPruned(const CBlockIndex* pblockindex); + +void CleanupBlockRevFiles(); + +/** Open a block file (blk?????.dat) */ +FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false); +/** Translation to a filesystem path */ +fs::path GetBlockPosFilename(const FlatFilePos& pos); + +/** Get block file info entry for one block file */ +CBlockFileInfo* GetBlockFileInfo(size_t n); + +/** Calculate the amount of disk space the block & undo files currently use */ +uint64_t CalculateCurrentUsage(); + +/** + * Actually unlink the specified files + */ +void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); + /** Functions for disk access for blocks */ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); @@ -32,6 +72,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); +bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams); FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index f8f0fff43f..38c1d29250 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -8,6 +8,7 @@ #include <coins.h> #include <crypto/muhash.h> #include <hash.h> +#include <index/coinstatsindex.h> #include <serialize.h> #include <uint256.h> #include <util/system.h> @@ -16,44 +17,22 @@ #include <map> // Database-independent metric indicating the UTXO set size -static uint64_t GetBogoSize(const CScript& scriptPubKey) +uint64_t GetBogoSize(const CScript& script_pub_key) { return 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + 2 /* scriptPubKey len */ + - scriptPubKey.size() /* scriptPubKey */; + script_pub_key.size() /* scriptPubKey */; } -static void ApplyHash(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) -{ - if (it == outputs.begin()) { - ss << hash; - ss << VARINT(it->second.nHeight * 2 + it->second.fCoinBase ? 1u : 0u); - } - - ss << VARINT(it->first + 1); - ss << it->second.out.scriptPubKey; - ss << VARINT_MODE(it->second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); - - if (it == std::prev(outputs.end())) { - ss << VARINT(0u); - } -} - -static void ApplyHash(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) {} - -static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) -{ - COutPoint outpoint = COutPoint(hash, it->first); - Coin coin = it->second; - +CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin) { CDataStream ss(SER_DISK, PROTOCOL_VERSION); ss << outpoint; ss << static_cast<uint32_t>(coin.nHeight * 2 + coin.fCoinBase); ss << coin.out; - muhash.Insert(MakeUCharSpan(ss)); + return ss; } //! Warning: be very careful when changing this! assumeutxo and UTXO snapshot @@ -68,14 +47,40 @@ static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& has //! It is also possible, though very unlikely, that a change in this //! construction could cause a previously invalid (and potentially malicious) //! UTXO snapshot to be considered valid. -template <typename T> -static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static void ApplyHash(CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + for (auto it = outputs.begin(); it != outputs.end(); ++it) { + if (it == outputs.begin()) { + ss << hash; + ss << VARINT(it->second.nHeight * 2 + it->second.fCoinBase ? 1u : 0u); + } + + ss << VARINT(it->first + 1); + ss << it->second.out.scriptPubKey; + ss << VARINT_MODE(it->second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); + + if (it == std::prev(outputs.end())) { + ss << VARINT(0u); + } + } +} + +static void ApplyHash(std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs) {} + +static void ApplyHash(MuHash3072& muhash, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + for (auto it = outputs.begin(); it != outputs.end(); ++it) { + COutPoint outpoint = COutPoint(hash, it->first); + Coin coin = it->second; + muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + } +} + +static void ApplyStats(CCoinsStats& stats, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); stats.nTransactions++; for (auto it = outputs.begin(); it != outputs.end(); ++it) { - ApplyHash(stats, hash_obj, hash, outputs, it); - stats.nTransactionOutputs++; stats.nTotalAmount += it->second.out.nValue; stats.nBogoSize += GetBogoSize(it->second.out.scriptPubKey); @@ -84,18 +89,25 @@ static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, con //! Calculate statistics about the unspent transaction output set template <typename T> -static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, T hash_obj, const std::function<void()>& interruption_point) +static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, T hash_obj, const std::function<void()>& interruption_point, const CBlockIndex* pindex) { - stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); assert(pcursor); - stats.hashBlock = pcursor->GetBestBlock(); - { - LOCK(cs_main); - assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman)); - const CBlockIndex* block = blockman.LookupBlockIndex(stats.hashBlock); - stats.nHeight = Assert(block)->nHeight; + if (!pindex) { + { + LOCK(cs_main); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman)); + pindex = blockman.LookupBlockIndex(view->GetBestBlock()); + } + } + stats.nHeight = Assert(pindex)->nHeight; + stats.hashBlock = pindex->GetBlockHash(); + + // Use CoinStatsIndex if it is requested and available and a hash_type of Muhash or None was requested + if ((stats.m_hash_type == CoinStatsHashType::MUHASH || stats.m_hash_type == CoinStatsHashType::NONE) && g_coin_stats_index && stats.index_requested) { + stats.index_used = true; + return g_coin_stats_index->LookUpStats(pindex, stats); } PrepareHash(hash_obj, stats); @@ -108,7 +120,8 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { if (!outputs.empty() && key.hash != prevkey) { - ApplyStats(stats, hash_obj, prevkey, outputs); + ApplyStats(stats, prevkey, outputs); + ApplyHash(hash_obj, prevkey, outputs); outputs.clear(); } prevkey = key.hash; @@ -120,7 +133,8 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& pcursor->Next(); } if (!outputs.empty()) { - ApplyStats(stats, hash_obj, prevkey, outputs); + ApplyStats(stats, prevkey, outputs); + ApplyHash(hash_obj, prevkey, outputs); } FinalizeHash(hash_obj, stats); @@ -129,19 +143,19 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& return true; } -bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, CoinStatsHashType hash_type, const std::function<void()>& interruption_point) +bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function<void()>& interruption_point, const CBlockIndex* pindex) { - switch (hash_type) { + switch (stats.m_hash_type) { case(CoinStatsHashType::HASH_SERIALIZED): { CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); - return GetUTXOStats(view, blockman, stats, ss, interruption_point); + return GetUTXOStats(view, blockman, stats, ss, interruption_point, pindex); } case(CoinStatsHashType::MUHASH): { MuHash3072 muhash; - return GetUTXOStats(view, blockman, stats, muhash, interruption_point); + return GetUTXOStats(view, blockman, stats, muhash, interruption_point, pindex); } case(CoinStatsHashType::NONE): { - return GetUTXOStats(view, blockman, stats, nullptr, interruption_point); + return GetUTXOStats(view, blockman, stats, nullptr, interruption_point, pindex); } } // no default case, so the compiler can warn about missing cases assert(false); diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 975651dcc4..8be256edc9 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -7,6 +7,9 @@ #define BITCOIN_NODE_COINSTATS_H #include <amount.h> +#include <chain.h> +#include <coins.h> +#include <streams.h> #include <uint256.h> #include <cstdint> @@ -23,6 +26,7 @@ enum class CoinStatsHashType { struct CCoinsStats { + CoinStatsHashType m_hash_type; int nHeight{0}; uint256 hashBlock{}; uint64_t nTransactions{0}; @@ -34,9 +38,31 @@ struct CCoinsStats //! The number of coins contained. uint64_t coins_count{0}; + + //! Signals if the coinstatsindex should be used (when available). + bool index_requested{true}; + //! Signals if the coinstatsindex was used to retrieve the statistics. + bool index_used{false}; + + // Following values are only available from coinstats index + CAmount total_subsidy{0}; + CAmount block_unspendable_amount{0}; + CAmount block_prevout_spent_amount{0}; + CAmount block_new_outputs_ex_coinbase_amount{0}; + CAmount block_coinbase_amount{0}; + CAmount unspendables_genesis_block{0}; + CAmount unspendables_bip30{0}; + CAmount unspendables_scripts{0}; + CAmount unspendables_unclaimed_rewards{0}; + + CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {} }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const CoinStatsHashType hash_type, const std::function<void()>& interruption_point = {}); +bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function<void()>& interruption_point = {}, const CBlockIndex* pindex = nullptr); + +uint64_t GetBogoSize(const CScript& script_pub_key); + +CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin); #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/node/context.h b/src/node/context.h index 2be9a584e6..06adb33a80 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -22,6 +22,7 @@ class PeerManager; namespace interfaces { class Chain; class ChainClient; +class Init; class WalletClient; } // namespace interfaces @@ -36,6 +37,8 @@ class WalletClient; //! any member functions. It should just be a collection of references that can //! be used without pulling in unwanted dependencies or functionality. struct NodeContext { + //! Init interface for initializing current process and connecting to other processes. + interfaces::Init* init{nullptr}; std::unique_ptr<CAddrMan> addrman; std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 691b2791d7..a1e7a71e2c 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -100,8 +100,6 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t // the mempool tracks locally submitted transactions to make a // best-effort of initial broadcast node.mempool->AddUnbroadcastTx(hashTx); - - LOCK(cs_main); node.peerman->RelayTransaction(hashTx, tx->GetWitnessHash()); } diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index fe78cb46bd..61292cdcc5 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -22,20 +22,15 @@ public: //! during snapshot load to estimate progress of UTXO set reconstruction. uint64_t m_coins_count = 0; - //! Necessary to "fake" the base nChainTx so that we can estimate progress during - //! initial block download for the assumeutxo chainstate. - unsigned int m_nchaintx = 0; - SnapshotMetadata() { } SnapshotMetadata( const uint256& base_blockhash, uint64_t coins_count, unsigned int nchaintx) : m_base_blockhash(base_blockhash), - m_coins_count(coins_count), - m_nchaintx(nchaintx) { } + m_coins_count(coins_count) { } - SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count, obj.m_nchaintx); } + SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); } }; #endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/pubkey.cpp b/src/pubkey.cpp index fc1624af25..a89988cc90 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -293,7 +293,7 @@ void CExtPubKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { bool CExtPubKey::Derive(CExtPubKey &out, unsigned int _nChild) const { out.nDepth = nDepth + 1; CKeyID id = pubkey.GetID(); - memcpy(&out.vchFingerprint[0], &id, 4); + memcpy(out.vchFingerprint, &id, 4); out.nChild = _nChild; return pubkey.Derive(out.pubkey, out.chaincode, _nChild, chaincode); } diff --git a/src/pubkey.h b/src/pubkey.h index 12514fc3c9..b653c202fc 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -101,7 +101,7 @@ public: } //! Construct a public key from a byte vector. - explicit CPubKey(const std::vector<unsigned char>& _vch) + explicit CPubKey(Span<const uint8_t> _vch) { Set(_vch.begin(), _vch.end()); } @@ -247,7 +247,7 @@ struct CExtPubKey { friend bool operator==(const CExtPubKey &a, const CExtPubKey &b) { return a.nDepth == b.nDepth && - memcmp(&a.vchFingerprint[0], &b.vchFingerprint[0], sizeof(vchFingerprint)) == 0 && + memcmp(a.vchFingerprint, b.vchFingerprint, sizeof(vchFingerprint)) == 0 && a.nChild == b.nChild && a.chaincode == b.chaincode && a.pubkey == b.pubkey; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index de71b7dea7..a30cac3504 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -319,11 +319,9 @@ void BitcoinApplication::parameterSetup() InitParameterInteraction(gArgs); } -void BitcoinApplication::InitializePruneSetting(bool prune) +void BitcoinApplication::InitPruneSetting(int64_t prune_MiB) { - // If prune is set, intentionally override existing prune size with - // the default size since this is called when choosing a new datadir. - optionsModel->SetPruneTargetGB(prune ? DEFAULT_PRUNE_TARGET_GB : 0, true); + optionsModel->SetPruneTargetGB(PruneMiBtoGB(prune_MiB), true); } void BitcoinApplication::requestInitialize() @@ -533,9 +531,9 @@ int GuiMain(int argc, char* argv[]) /// 5. Now that settings and translations are available, ask user for data directory // User language is set up: pick a data directory bool did_show_intro = false; - bool prune = false; // Intro dialog prune check box + int64_t prune_MiB = 0; // Intro dialog prune configuration // Gracefully exit if the user cancels - if (!Intro::showIfNeeded(did_show_intro, prune)) return EXIT_SUCCESS; + if (!Intro::showIfNeeded(did_show_intro, prune_MiB)) return EXIT_SUCCESS; /// 6. Determine availability of data directory and parse bitcoin.conf /// - Do not call GetDataDir(true) before this step finishes @@ -617,7 +615,7 @@ int GuiMain(int argc, char* argv[]) if (did_show_intro) { // Store intro dialog settings other than datadir (network specific) - app.InitializePruneSetting(prune); + app.InitPruneSetting(prune_MiB); } if (gArgs.GetBoolArg("-splash", DEFAULT_SPLASHSCREEN) && !gArgs.GetBoolArg("-min", false)) diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 5fd6bd607f..f9fab0534b 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -68,7 +68,7 @@ public: /// Create options model void createOptionsModel(bool resetSettings); /// Initialize prune setting - void InitializePruneSetting(bool prune); + void InitPruneSetting(int64_t prune_MiB); /// Create main window void createWindow(const NetworkStyle *networkStyle); /// Create splash screen diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index f2c555de52..04161020b2 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -8,6 +8,7 @@ #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/peertablemodel.h> +#include <qt/peertablesortproxy.h> #include <clientversion.h> #include <interfaces/handler.h> @@ -38,7 +39,11 @@ ClientModel::ClientModel(interfaces::Node& node, OptionsModel *_optionsModel, QO { cachedBestHeaderHeight = -1; cachedBestHeaderTime = -1; + peerTableModel = new PeerTableModel(m_node, this); + m_peer_table_sort_proxy = new PeerTableSortProxy(this); + m_peer_table_sort_proxy->setSourceModel(peerTableModel); + banTableModel = new BanTableModel(m_node, this); QTimer* timer = new QTimer; @@ -184,6 +189,11 @@ PeerTableModel *ClientModel::getPeerTableModel() return peerTableModel; } +PeerTableSortProxy* ClientModel::peerTableSortProxy() +{ + return m_peer_table_sort_proxy; +} + BanTableModel *ClientModel::getBanTableModel() { return banTableModel; diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 7ac4cc040b..7a199ef19c 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -17,6 +17,7 @@ class BanTableModel; class CBlockIndex; class OptionsModel; class PeerTableModel; +class PeerTableSortProxy; enum class SynchronizationState; namespace interfaces { @@ -54,6 +55,7 @@ public: interfaces::Node& node() const { return m_node; } OptionsModel *getOptionsModel(); PeerTableModel *getPeerTableModel(); + PeerTableSortProxy* peerTableSortProxy(); BanTableModel *getBanTableModel(); //! Return number of connections, default is in- and outbound (total) @@ -96,6 +98,7 @@ private: std::unique_ptr<interfaces::Handler> m_handler_notify_header_tip; OptionsModel *optionsModel; PeerTableModel *peerTableModel; + PeerTableSortProxy* m_peer_table_sort_proxy{nullptr}; BanTableModel *banTableModel; //! A thread to interact with m_node asynchronously diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 13687a6510..61ecea1cac 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -923,6 +923,9 @@ <property name="tabKeyNavigation"> <bool>false</bool> </property> + <property name="alternatingRowColors"> + <bool>true</bool> + </property> <property name="textElideMode"> <enum>Qt::ElideMiddle</enum> </property> @@ -984,6 +987,9 @@ <property name="tabKeyNavigation"> <bool>false</bool> </property> + <property name="alternatingRowColors"> + <bool>true</bool> + </property> <property name="sortingEnabled"> <bool>true</bool> </property> diff --git a/src/qt/forms/intro.ui b/src/qt/forms/intro.ui index f27a4ebe44..a1e94f99e6 100644 --- a/src/qt/forms/intro.ui +++ b/src/qt/forms/intro.ui @@ -7,7 +7,7 @@ <x>0</x> <y>0</y> <width>674</width> - <height>415</height> + <height>447</height> </rect> </property> <property name="windowTitle"> @@ -211,16 +211,6 @@ </widget> </item> <item> - <widget class="QCheckBox" name="prune"> - <property name="toolTip"> - <string>Reverting this setting requires re-downloading the entire blockchain. It is faster to download the full chain first and prune it later. Disables some advanced features.</string> - </property> - <property name="text"> - <string></string> - </property> - </widget> - </item> - <item> <widget class="QLabel" name="lblExplanation2"> <property name="text"> <string>This initial synchronisation is very demanding, and may expose hardware problems with your computer that had previously gone unnoticed. Each time you run %1, it will continue downloading where it left off.</string> @@ -241,6 +231,47 @@ </widget> </item> <item> + <layout class="QHBoxLayout" name="pruneOptLayout"> + <item> + <widget class="QCheckBox" name="prune"> + <property name="text"> + <string>Limit block chain storage to</string> + </property> + <property name="toolTip"> + <string>Reverting this setting requires re-downloading the entire blockchain. It is faster to download the full chain first and prune it later. Disables some advanced features.</string> + </property> + </widget> + </item> + <item> + <widget class="QSpinBox" name="pruneGB"> + <property name="suffix"> + <string> GB</string> + </property> + </widget> + </item> + <item> + <widget class="QLabel" name="lblPruneSuffix"> + <property name="buddy"> + <cstring>pruneGB</cstring> + </property> + </widget> + </item> + <item> + <spacer name="horizontalSpacer_2"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + </layout> + </item> + <item> <spacer name="verticalSpacer"> <property name="orientation"> <enum>Qt::Vertical</enum> diff --git a/src/qt/forms/modaloverlay.ui b/src/qt/forms/modaloverlay.ui index d2e7ca8f06..83b6a59aa0 100644 --- a/src/qt/forms/modaloverlay.ui +++ b/src/qt/forms/modaloverlay.ui @@ -271,16 +271,6 @@ QLabel { color: rgb(40,40,40); }</string> </property> </widget> </item> - <item> - <widget class="QProgressBar" name="progressBar"> - <property name="value"> - <number>24</number> - </property> - <property name="format"> - <string/> - </property> - </widget> - </item> </layout> </item> <item row="4" column="0"> diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index aa6b2665fa..ed39307fd7 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -17,6 +17,7 @@ #include <interfaces/node.h> #include <util/system.h> +#include <validation.h> #include <QFileDialog> #include <QSettings> @@ -139,17 +140,26 @@ Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_si ); ui->lblExplanation2->setText(ui->lblExplanation2->text().arg(PACKAGE_NAME)); + const int min_prune_target_GB = std::ceil(MIN_DISK_SPACE_FOR_BLOCK_FILES / 1e9); + ui->pruneGB->setRange(min_prune_target_GB, std::numeric_limits<int>::max()); if (gArgs.GetArg("-prune", 0) > 1) { // -prune=1 means enabled, above that it's a size in MiB ui->prune->setChecked(true); ui->prune->setEnabled(false); } - ui->prune->setText(tr("Discard blocks after verification, except most recent %1 GB (prune)").arg(m_prune_target_gb)); + ui->pruneGB->setValue(m_prune_target_gb); + ui->pruneGB->setToolTip(ui->prune->toolTip()); + ui->lblPruneSuffix->setToolTip(ui->prune->toolTip()); UpdatePruneLabels(ui->prune->isChecked()); connect(ui->prune, &QCheckBox::toggled, [this](bool prune_checked) { UpdatePruneLabels(prune_checked); UpdateFreeSpaceLabel(); }); + connect(ui->pruneGB, QOverload<int>::of(&QSpinBox::valueChanged), [this](int prune_GB) { + m_prune_target_gb = prune_GB; + UpdatePruneLabels(ui->prune->isChecked()); + UpdateFreeSpaceLabel(); + }); startThread(); } @@ -182,7 +192,17 @@ void Intro::setDataDirectory(const QString &dataDir) } } -bool Intro::showIfNeeded(bool& did_show_intro, bool& prune) +int64_t Intro::getPruneMiB() const +{ + switch (ui->prune->checkState()) { + case Qt::Checked: + return PruneGBtoMiB(m_prune_target_gb); + case Qt::Unchecked: default: + return 0; + } +} + +bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) { did_show_intro = false; @@ -233,7 +253,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, bool& prune) } // Additional preferences: - prune = intro.ui->prune->isChecked(); + prune_MiB = intro.getPruneMiB(); settings.setValue("strDataDir", dataDir); settings.setValue("fReset", false); @@ -361,6 +381,11 @@ void Intro::UpdatePruneLabels(bool prune_checked) storageRequiresMsg = tr("Approximately %1 GB of data will be stored in this directory."); } ui->lblExplanation3->setVisible(prune_checked); + ui->pruneGB->setEnabled(prune_checked); + static constexpr uint64_t nPowTargetSpacing = 10 * 60; // from chainparams, which we don't have at this stage + static constexpr uint32_t expected_block_data_size = 2250000; // includes undo data + const uint64_t expected_backup_days = m_prune_target_gb * 1e9 / (uint64_t(expected_block_data_size) * 86400 / nPowTargetSpacing); + ui->lblPruneSuffix->setText(tr("(sufficient to restore backups %n day(s) old)", "block chain pruning", expected_backup_days)); ui->sizeWarningLabel->setText( tr("%1 will download and store a copy of the Bitcoin block chain.").arg(PACKAGE_NAME) + " " + storageRequiresMsg.arg(m_required_space_gb) + " " + diff --git a/src/qt/intro.h b/src/qt/intro.h index 51f42de7ac..88fe2b722d 100644 --- a/src/qt/intro.h +++ b/src/qt/intro.h @@ -36,6 +36,7 @@ public: QString getDataDirectory(); void setDataDirectory(const QString &dataDir); + int64_t getPruneMiB() const; /** * Determine data directory. Let the user choose if the current one doesn't exist. @@ -47,7 +48,7 @@ public: * @note do NOT call global GetDataDir() before calling this function, this * will cause the wrong path to be cached. */ - static bool showIfNeeded(bool& did_show_intro, bool& prune); + static bool showIfNeeded(bool& did_show_intro, int64_t& prune_MiB); Q_SIGNALS: void requestCheck(); @@ -72,7 +73,7 @@ private: //! Total required space (in GB) depending on user choice (prune or not prune). int64_t m_required_space_gb{0}; uint64_t m_bytes_available{0}; - const int64_t m_prune_target_gb; + int64_t m_prune_target_gb; void startThread(); void checkPath(const QString &dataDir); diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 8070aa627c..6363fe2edf 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -134,7 +134,6 @@ void ModalOverlay::tipUpdate(int count, const QDateTime& blockDate, double nVeri // show the percentage done according to nVerificationProgress ui->percentageProgress->setText(QString::number(nVerificationProgress*100, 'f', 2)+"%"); - ui->progressBar->setValue(nVerificationProgress*100); if (!bestHeaderDate.isValid()) // not syncing diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 3459bf4cf8..6c4e326011 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -11,56 +11,19 @@ #include <utility> -#include <QDebug> #include <QList> #include <QTimer> -bool NodeLessThan::operator()(const CNodeCombinedStats &left, const CNodeCombinedStats &right) const -{ - const CNodeStats *pLeft = &(left.nodeStats); - const CNodeStats *pRight = &(right.nodeStats); - - if (order == Qt::DescendingOrder) - std::swap(pLeft, pRight); - - switch (static_cast<PeerTableModel::ColumnIndex>(column)) { - case PeerTableModel::NetNodeId: - return pLeft->nodeid < pRight->nodeid; - case PeerTableModel::Address: - return pLeft->addrName.compare(pRight->addrName) < 0; - case PeerTableModel::ConnectionType: - return pLeft->m_conn_type < pRight->m_conn_type; - case PeerTableModel::Network: - return pLeft->m_network < pRight->m_network; - case PeerTableModel::Ping: - return pLeft->m_min_ping_time < pRight->m_min_ping_time; - case PeerTableModel::Sent: - return pLeft->nSendBytes < pRight->nSendBytes; - case PeerTableModel::Received: - return pLeft->nRecvBytes < pRight->nRecvBytes; - case PeerTableModel::Subversion: - return pLeft->cleanSubVer.compare(pRight->cleanSubVer) < 0; - } // no default case, so the compiler can warn about missing cases - assert(false); -} - // private implementation class PeerTablePriv { public: /** Local cache of peer information */ QList<CNodeCombinedStats> cachedNodeStats; - /** Column to sort nodes by (default to unsorted) */ - int sortColumn{-1}; - /** Order (ascending or descending) to sort nodes by */ - Qt::SortOrder sortOrder; - /** Index of rows by node ID */ - std::map<NodeId, int> mapNodeRows; /** Pull a full list of peers from vNodes into our cache */ void refreshPeers(interfaces::Node& node) { - { cachedNodeStats.clear(); interfaces::Node::NodesStats nodes_stats; @@ -74,17 +37,6 @@ public: stats.nodeStateStats = std::get<2>(node_stats); cachedNodeStats.append(stats); } - } - - if (sortColumn >= 0) - // sort cacheNodeStats (use stable sort to prevent rows jumping around unnecessarily) - std::stable_sort(cachedNodeStats.begin(), cachedNodeStats.end(), NodeLessThan(sortColumn, sortOrder)); - - // build index map - mapNodeRows.clear(); - int row = 0; - for (const CNodeCombinedStats& stats : cachedNodeStats) - mapNodeRows.insert(std::pair<NodeId, int>(stats.nodeStats.nodeid, row++)); } int size() const @@ -194,10 +146,7 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const } // no default case, so the compiler can warn about missing cases assert(false); } else if (role == StatsRole) { - switch (index.column()) { - case NetNodeId: return QVariant::fromValue(rec); - default: return QVariant(); - } + return QVariant::fromValue(rec); } return QVariant(); @@ -239,19 +188,3 @@ void PeerTableModel::refresh() priv->refreshPeers(m_node); Q_EMIT layoutChanged(); } - -int PeerTableModel::getRowByNodeId(NodeId nodeid) -{ - std::map<NodeId, int>::iterator it = priv->mapNodeRows.find(nodeid); - if (it == priv->mapNodeRows.end()) - return -1; - - return it->second; -} - -void PeerTableModel::sort(int column, Qt::SortOrder order) -{ - priv->sortColumn = column; - priv->sortOrder = order; - refresh(); -} diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index 0823235ec0..9c7bc25da2 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -30,18 +30,6 @@ struct CNodeCombinedStats { }; Q_DECLARE_METATYPE(CNodeCombinedStats*) -class NodeLessThan -{ -public: - NodeLessThan(int nColumn, Qt::SortOrder fOrder) : - column(nColumn), order(fOrder) {} - bool operator()(const CNodeCombinedStats &left, const CNodeCombinedStats &right) const; - -private: - int column; - Qt::SortOrder order; -}; - /** Qt model providing information about connected peers, similar to the "getpeerinfo" RPC call. Used by the rpc console UI. @@ -53,7 +41,6 @@ class PeerTableModel : public QAbstractTableModel public: explicit PeerTableModel(interfaces::Node& node, QObject* parent); ~PeerTableModel(); - int getRowByNodeId(NodeId nodeid); void startAutoRefresh(); void stopAutoRefresh(); @@ -80,7 +67,6 @@ public: QVariant headerData(int section, Qt::Orientation orientation, int role) const override; QModelIndex index(int row, int column, const QModelIndex &parent) const override; Qt::ItemFlags flags(const QModelIndex &index) const override; - void sort(int column, Qt::SortOrder order) override; /*@}*/ public Q_SLOTS: diff --git a/src/qt/peertablesortproxy.cpp b/src/qt/peertablesortproxy.cpp new file mode 100644 index 0000000000..78932da8d4 --- /dev/null +++ b/src/qt/peertablesortproxy.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <qt/peertablesortproxy.h> + +#include <qt/peertablemodel.h> +#include <util/check.h> + +#include <QModelIndex> +#include <QString> +#include <QVariant> + +PeerTableSortProxy::PeerTableSortProxy(QObject* parent) + : QSortFilterProxyModel(parent) +{ +} + +bool PeerTableSortProxy::lessThan(const QModelIndex& left_index, const QModelIndex& right_index) const +{ + const CNodeStats left_stats = Assert(sourceModel()->data(left_index, PeerTableModel::StatsRole).value<CNodeCombinedStats*>())->nodeStats; + const CNodeStats right_stats = Assert(sourceModel()->data(right_index, PeerTableModel::StatsRole).value<CNodeCombinedStats*>())->nodeStats; + + switch (static_cast<PeerTableModel::ColumnIndex>(left_index.column())) { + case PeerTableModel::NetNodeId: + return left_stats.nodeid < right_stats.nodeid; + case PeerTableModel::Address: + return left_stats.addrName.compare(right_stats.addrName) < 0; + case PeerTableModel::ConnectionType: + return left_stats.m_conn_type < right_stats.m_conn_type; + case PeerTableModel::Network: + return left_stats.m_network < right_stats.m_network; + case PeerTableModel::Ping: + return left_stats.m_min_ping_time < right_stats.m_min_ping_time; + case PeerTableModel::Sent: + return left_stats.nSendBytes < right_stats.nSendBytes; + case PeerTableModel::Received: + return left_stats.nRecvBytes < right_stats.nRecvBytes; + case PeerTableModel::Subversion: + return left_stats.cleanSubVer.compare(right_stats.cleanSubVer) < 0; + } // no default case, so the compiler can warn about missing cases + assert(false); +} diff --git a/src/qt/peertablesortproxy.h b/src/qt/peertablesortproxy.h new file mode 100644 index 0000000000..1879f6b400 --- /dev/null +++ b/src/qt/peertablesortproxy.h @@ -0,0 +1,25 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_PEERTABLESORTPROXY_H +#define BITCOIN_QT_PEERTABLESORTPROXY_H + +#include <QSortFilterProxyModel> + +QT_BEGIN_NAMESPACE +class QModelIndex; +QT_END_NAMESPACE + +class PeerTableSortProxy : public QSortFilterProxyModel +{ + Q_OBJECT + +public: + explicit PeerTableSortProxy(QObject* parent = nullptr); + +protected: + bool lessThan(const QModelIndex& left_index, const QModelIndex& right_index) const override; +}; + +#endif // BITCOIN_QT_PEERTABLESORTPROXY_H diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 85412ca75b..006f60e7a1 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -9,13 +9,14 @@ #include <qt/rpcconsole.h> #include <qt/forms/ui_debugwindow.h> +#include <chainparams.h> +#include <interfaces/node.h> +#include <netbase.h> #include <qt/bantablemodel.h> #include <qt/clientmodel.h> +#include <qt/peertablesortproxy.h> #include <qt/platformstyle.h> #include <qt/walletmodel.h> -#include <chainparams.h> -#include <interfaces/node.h> -#include <netbase.h> #include <rpc/client.h> #include <rpc/server.h> #include <util/strencodings.h> @@ -606,7 +607,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ connect(model, &ClientModel::mempoolSizeChanged, this, &RPCConsole::setMempoolSize); // set up peer table - ui->peerWidget->setModel(model->getPeerTableModel()); + ui->peerWidget->setModel(model->peerTableSortProxy()); ui->peerWidget->verticalHeader()->hide(); ui->peerWidget->setSelectionBehavior(QAbstractItemView::SelectRows); ui->peerWidget->setSelectionMode(QAbstractItemView::ExtendedSelection); @@ -627,10 +628,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ // peer table signal handling - update peer details when selecting new node connect(ui->peerWidget->selectionModel(), &QItemSelectionModel::selectionChanged, this, &RPCConsole::updateDetailWidget); - // peer table signal handling - update peer details when new nodes are added to the model - connect(model->getPeerTableModel(), &PeerTableModel::layoutChanged, this, &RPCConsole::peerLayoutChanged); - // peer table signal handling - cache selected node ids - connect(model->getPeerTableModel(), &PeerTableModel::layoutAboutToBeChanged, this, &RPCConsole::peerLayoutAboutToChange); + connect(model->getPeerTableModel(), &PeerTableModel::layoutChanged, this, &RPCConsole::updateDetailWidget); // set up ban table ui->banlistWidget->setModel(model->getBanTableModel()); @@ -1014,67 +1012,6 @@ void RPCConsole::updateTrafficStats(quint64 totalBytesIn, quint64 totalBytesOut) ui->lblBytesOut->setText(GUIUtil::formatBytes(totalBytesOut)); } -void RPCConsole::peerLayoutAboutToChange() -{ - cachedNodeids.clear(); - for (const QModelIndex& peer : GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId)) { - const auto stats = peer.data(PeerTableModel::StatsRole).value<CNodeCombinedStats*>(); - cachedNodeids.append(stats->nodeStats.nodeid); - } -} - -void RPCConsole::peerLayoutChanged() -{ - if (!clientModel || !clientModel->getPeerTableModel()) - return; - - bool fUnselect = false; - bool fReselect = false; - - if (cachedNodeids.empty()) // no node selected yet - return; - - // find the currently selected row - int selectedRow = -1; - QModelIndexList selectedModelIndex = ui->peerWidget->selectionModel()->selectedIndexes(); - if (!selectedModelIndex.isEmpty()) { - selectedRow = selectedModelIndex.first().row(); - } - - // check if our detail node has a row in the table (it may not necessarily - // be at selectedRow since its position can change after a layout change) - int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeids.first()); - - if (detailNodeRow < 0) - { - // detail node disappeared from table (node disconnected) - fUnselect = true; - } - else - { - if (detailNodeRow != selectedRow) - { - // detail node moved position - fUnselect = true; - fReselect = true; - } - } - - if (fUnselect && selectedRow >= 0) { - clearSelectedNode(); - } - - if (fReselect) - { - for(int i = 0; i < cachedNodeids.size(); i++) - { - ui->peerWidget->selectRow(clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeids.at(i))); - } - } - - updateDetailWidget(); -} - void RPCConsole::updateDetailWidget() { const QList<QModelIndex> selected_peers = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index b9806e40c9..5182d60a0d 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -118,10 +118,6 @@ public Q_SLOTS: void browseHistory(int offset); /** Scroll console view to end */ void scrollToEnd(); - /** Handle selection caching before update */ - void peerLayoutAboutToChange(); - /** Handle updated peer information */ - void peerLayoutChanged(); /** Disconnect a selected node on the Peers tab */ void disconnectSelectedNode(); /** Ban a selected node on the Peers tab */ diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index f0e720617c..e1a4faa266 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -271,7 +271,7 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa m_current_transaction = std::make_unique<WalletModelTransaction>(recipients); WalletModel::SendCoinsReturn prepareStatus; - updateCoinControlState(*m_coin_control); + updateCoinControlState(); prepareStatus = model->prepareTransaction(*m_current_transaction, *m_coin_control); @@ -740,19 +740,19 @@ void SendCoinsDialog::updateFeeMinimizedLabel() } } -void SendCoinsDialog::updateCoinControlState(CCoinControl& ctrl) +void SendCoinsDialog::updateCoinControlState() { if (ui->radioCustomFee->isChecked()) { - ctrl.m_feerate = CFeeRate(ui->customFee->value()); + m_coin_control->m_feerate = CFeeRate(ui->customFee->value()); } else { - ctrl.m_feerate.reset(); + m_coin_control->m_feerate.reset(); } // Avoid using global defaults when sending money from the GUI // Either custom fee will be used or if not selected, the confirmation target from dropdown box - ctrl.m_confirm_target = getConfTargetForIndex(ui->confTargetSelector->currentIndex()); - ctrl.m_signal_bip125_rbf = ui->optInRBF->isChecked(); + m_coin_control->m_confirm_target = getConfTargetForIndex(ui->confTargetSelector->currentIndex()); + m_coin_control->m_signal_bip125_rbf = ui->optInRBF->isChecked(); // Include watch-only for wallets without private key - ctrl.fAllowWatchOnly = model->wallet().privateKeysDisabled(); + m_coin_control->fAllowWatchOnly = model->wallet().privateKeysDisabled(); } void SendCoinsDialog::updateNumberOfBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers, SynchronizationState sync_state) { @@ -765,7 +765,7 @@ void SendCoinsDialog::updateSmartFeeLabel() { if(!model || !model->getOptionsModel()) return; - updateCoinControlState(*m_coin_control); + updateCoinControlState(); m_coin_control->m_feerate.reset(); // Explicitly use only fee estimation rate for smart fee labels int returned_target; FeeReason reason; @@ -839,8 +839,9 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) { ui->frameCoinControl->setVisible(checked); - if (!checked && model) // coin control features disabled - m_coin_control->SetNull(); + if (!checked && model) { // coin control features disabled + m_coin_control = std::make_unique<CCoinControl>(); + } coinControlUpdateLabels(); } @@ -928,7 +929,7 @@ void SendCoinsDialog::coinControlUpdateLabels() if (!model || !model->getOptionsModel()) return; - updateCoinControlState(*m_coin_control); + updateCoinControlState(); // set pay amounts CoinControlDialog::payAmounts.clear(); diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 3e276201ba..33736f8095 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -76,8 +76,7 @@ private: // Format confirmation message bool PrepareSendText(QString& question_string, QString& informative_text, QString& detailed_text); void updateFeeMinimizedLabel(); - // Update the passed in CCoinControl with state from the GUI - void updateCoinControlState(CCoinControl& ctrl); + void updateCoinControlState(); private Q_SLOTS: void sendButtonClicked(bool checked); diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 9e6e98b274..df3b85ea71 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -35,14 +35,16 @@ static RPCHelpMan rpcNestedTest_rpc() } static const CRPCCommand vRPCCommands[] = { - {"test", &rpcNestedTest_rpc}, + {"rpcNestedTest", &rpcNestedTest_rpc}, }; void RPCNestedTests::rpcNestedTests() { // do some test setup // could be moved to a more generic place when we add more tests on QT level - tableRPC.appendCommand("rpcNestedTest", &vRPCCommands[0]); + for (const auto& c : vRPCCommands) { + tableRPC.appendCommand(c.name, &c); + } TestingSetup test; m_node.setContext(&test.m_node); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index c6dd0c2ad6..cc6db8d33e 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -248,7 +248,7 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << *newTx; - transaction_array.append((const char*)&(ssTx[0]), ssTx.size()); + transaction_array.append((const char*)ssTx.data(), ssTx.size()); } // Add addresses / update labels that we've sent to the address book, diff --git a/src/random.cpp b/src/random.cpp index 9900825abb..174f4cef31 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -628,7 +628,7 @@ std::vector<unsigned char> FastRandomContext::randbytes(size_t len) if (requires_seed) RandomSeed(); std::vector<unsigned char> ret(len); if (len > 0) { - rng.Keystream(&ret[0], len); + rng.Keystream(ret.data(), len); } return ret; } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e7fd97ee1f..81cb802357 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -14,6 +14,7 @@ #include <core_io.h> #include <hash.h> #include <index/blockfilterindex.h> +#include <index/coinstatsindex.h> #include <node/blockstorage.h> #include <node/coinstats.h> #include <node/context.h> @@ -139,6 +140,33 @@ static int ComputeNextBlockAndDepth(const CBlockIndex* tip, const CBlockIndex* b return blockindex == tip ? 1 : -1; } +CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) { + LOCK(::cs_main); + CChain& active_chain = chainman.ActiveChain(); + + if (param.isNum()) { + const int height{param.get_int()}; + if (height < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d is negative", height)); + } + const int current_tip{active_chain.Height()}; + if (height > current_tip) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d after current tip %d", height, current_tip)); + } + + return active_chain[height]; + } else { + const uint256 hash{ParseHashV(param, "hash_or_height")}; + CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); + + if (!pindex) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + + return pindex; + } +} + UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex) { // Serialize passed information without accessing chain state of the active chain! @@ -1068,50 +1096,88 @@ static RPCHelpMan gettxoutsetinfo() { return RPCHelpMan{"gettxoutsetinfo", "\nReturns statistics about the unspent transaction output set.\n" - "Note this call may take some time.\n", + "Note this call may take some time if you are not using coinstatsindex.\n", { {"hash_type", RPCArg::Type::STR, RPCArg::Default{"hash_serialized_2"}, "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."}, + {"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The block hash or height of the target height (only available with coinstatsindex).", "", {"", "string or numeric"}}, + {"use_index", RPCArg::Type::BOOL, RPCArg::Default{true}, "Use coinstatsindex, if available."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", { {RPCResult::Type::NUM, "height", "The block height (index) of the returned statistics"}, {RPCResult::Type::STR_HEX, "bestblock", "The hash of the block at which these statistics are calculated"}, - {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs"}, {RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs"}, - {RPCResult::Type::NUM, "bogosize", "A meaningless metric for UTXO set size"}, + {RPCResult::Type::NUM, "bogosize", "Database-independent, meaningless metric indicating the UTXO set size"}, {RPCResult::Type::STR_HEX, "hash_serialized_2", /* optional */ true, "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, {RPCResult::Type::STR_HEX, "muhash", /* optional */ true, "The serialized hash (only present if 'muhash' hash_type is chosen)"}, - {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"}, + {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs (not available when coinstatsindex is used)"}, + {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk (not available when coinstatsindex is used)"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of coins in the UTXO set"}, + {RPCResult::Type::STR_AMOUNT, "total_unspendable_amount", "The total amount of coins permanently excluded from the UTXO set (only available if coinstatsindex is used)"}, + {RPCResult::Type::OBJ, "block_info", "Info on amounts in the block at this block height (only available if coinstatsindex is used)", + { + {RPCResult::Type::STR_AMOUNT, "prevout_spent", ""}, + {RPCResult::Type::STR_AMOUNT, "coinbase", ""}, + {RPCResult::Type::STR_AMOUNT, "new_outputs_ex_coinbase", ""}, + {RPCResult::Type::STR_AMOUNT, "unspendable", ""}, + {RPCResult::Type::OBJ, "unspendables", "Detailed view of the unspendable categories", + { + {RPCResult::Type::STR_AMOUNT, "genesis_block", ""}, + {RPCResult::Type::STR_AMOUNT, "bip30", "Transactions overridden by duplicates (no longer possible with BIP30)"}, + {RPCResult::Type::STR_AMOUNT, "scripts", "Amounts sent to scripts that are unspendable (for example OP_RETURN outputs)"}, + {RPCResult::Type::STR_AMOUNT, "unclaimed_rewards", "Fee rewards that miners did not claim in their coinbase transaction"}, + }} + }}, }}, RPCExamples{ - HelpExampleCli("gettxoutsetinfo", "") - + HelpExampleRpc("gettxoutsetinfo", "") + HelpExampleCli("gettxoutsetinfo", "") + + HelpExampleCli("gettxoutsetinfo", R"("none")") + + HelpExampleCli("gettxoutsetinfo", R"("none" 1000)") + + HelpExampleCli("gettxoutsetinfo", R"("none" '"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"')") + + HelpExampleRpc("gettxoutsetinfo", "") + + HelpExampleRpc("gettxoutsetinfo", R"("none")") + + HelpExampleRpc("gettxoutsetinfo", R"("none", 1000)") + + HelpExampleRpc("gettxoutsetinfo", R"("none", "00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09")") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { UniValue ret(UniValue::VOBJ); - CCoinsStats stats; + CBlockIndex* pindex{nullptr}; + const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; + CCoinsStats stats{hash_type}; + stats.index_requested = request.params[2].isNull() || request.params[2].get_bool(); + NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); CChainState& active_chainstate = chainman.ActiveChainstate(); active_chainstate.ForceFlushStateToDisk(); - const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; - CCoinsView* coins_view; BlockManager* blockman; { LOCK(::cs_main); coins_view = &active_chainstate.CoinsDB(); blockman = &active_chainstate.m_blockman; + pindex = blockman->LookupBlockIndex(coins_view->GetBestBlock()); } - if (GetUTXOStats(coins_view, *blockman, stats, hash_type, node.rpc_interruption_point)) { + + if (!request.params[1].isNull()) { + if (!g_coin_stats_index) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Querying specific block heights requires coinstatsindex"); + } + + if (stats.m_hash_type == CoinStatsHashType::HASH_SERIALIZED) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "hash_serialized_2 hash type cannot be queried for a specific block"); + } + + pindex = ParseHashOrHeight(request.params[1], chainman); + } + + if (GetUTXOStats(coins_view, *blockman, stats, node.rpc_interruption_point, pindex)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); - ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); ret.pushKV("bogosize", (int64_t)stats.nBogoSize); if (hash_type == CoinStatsHashType::HASH_SERIALIZED) { @@ -1120,9 +1186,42 @@ static RPCHelpMan gettxoutsetinfo() if (hash_type == CoinStatsHashType::MUHASH) { ret.pushKV("muhash", stats.hashSerialized.GetHex()); } - ret.pushKV("disk_size", stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); + if (!stats.index_used) { + ret.pushKV("transactions", static_cast<int64_t>(stats.nTransactions)); + ret.pushKV("disk_size", stats.nDiskSize); + } else { + ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.block_unspendable_amount)); + + CCoinsStats prev_stats{hash_type}; + + if (pindex->nHeight > 0) { + GetUTXOStats(coins_view, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), prev_stats, node.rpc_interruption_point, pindex->pprev); + } + + UniValue block_info(UniValue::VOBJ); + block_info.pushKV("prevout_spent", ValueFromAmount(stats.block_prevout_spent_amount - prev_stats.block_prevout_spent_amount)); + block_info.pushKV("coinbase", ValueFromAmount(stats.block_coinbase_amount - prev_stats.block_coinbase_amount)); + block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.block_new_outputs_ex_coinbase_amount - prev_stats.block_new_outputs_ex_coinbase_amount)); + block_info.pushKV("unspendable", ValueFromAmount(stats.block_unspendable_amount - prev_stats.block_unspendable_amount)); + + UniValue unspendables(UniValue::VOBJ); + unspendables.pushKV("genesis_block", ValueFromAmount(stats.unspendables_genesis_block - prev_stats.unspendables_genesis_block)); + unspendables.pushKV("bip30", ValueFromAmount(stats.unspendables_bip30 - prev_stats.unspendables_bip30)); + unspendables.pushKV("scripts", ValueFromAmount(stats.unspendables_scripts - prev_stats.unspendables_scripts)); + unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.unspendables_unclaimed_rewards - prev_stats.unspendables_unclaimed_rewards)); + block_info.pushKV("unspendables", unspendables); + + ret.pushKV("block_info", block_info); + } } else { + if (g_coin_stats_index) { + const IndexSummary summary{g_coin_stats_index->GetSummary()}; + + if (!summary.synced) { + throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to read UTXO set because coinstatsindex is still syncing. Current height: %d", summary.best_block_height)); + } + } throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } return ret; @@ -1239,7 +1338,8 @@ static RPCHelpMan verifychain() LOCK(cs_main); CChainState& active_chainstate = chainman.ActiveChainstate(); - return CVerifyDB().VerifyDB(Params(), active_chainstate, &active_chainstate.CoinsTip(), check_level, check_depth); + return CVerifyDB().VerifyDB( + active_chainstate, Params(), active_chainstate.CoinsTip(), check_level, check_depth); }, }; } @@ -1909,31 +2009,7 @@ static RPCHelpMan getblockstats() { ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); - CChain& active_chain = chainman.ActiveChain(); - - CBlockIndex* pindex; - if (request.params[0].isNum()) { - const int height = request.params[0].get_int(); - const int current_tip = active_chain.Height(); - if (height < 0) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d is negative", height)); - } - if (height > current_tip) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d after current tip %d", height, current_tip)); - } - - pindex = active_chain[height]; - } else { - const uint256 hash(ParseHashV(request.params[0], "hash_or_height")); - pindex = chainman.m_blockman.LookupBlockIndex(hash); - if (!pindex) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - } - if (!active_chain.Contains(pindex)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Block is not in chain %s", Params().NetworkIDString())); - } - } - + CBlockIndex* pindex{ParseHashOrHeight(request.params[0], chainman)}; CHECK_NONFATAL(pindex != nullptr); std::set<std::string> stats; @@ -2491,7 +2567,7 @@ static RPCHelpMan dumptxoutset() UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile) { std::unique_ptr<CCoinsViewCursor> pcursor; - CCoinsStats stats; + CCoinsStats stats{CoinStatsHashType::NONE}; CBlockIndex* tip; { @@ -2511,7 +2587,7 @@ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFil chainstate.ForceFlushStateToDisk(); - if (!GetUTXOStats(&chainstate.CoinsDB(), chainstate.m_blockman, stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) { + if (!GetUTXOStats(&chainstate.CoinsDB(), chainstate.m_blockman, stats, node.rpc_interruption_point)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 2b593cd10b..9c8582c7a3 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -127,6 +127,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxout", 1, "n" }, { "gettxout", 2, "include_mempool" }, { "gettxoutproof", 0, "txids" }, + { "gettxoutsetinfo", 1, "hash_or_height" }, + { "gettxoutsetinfo", 2, "use_index"}, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, { "send", 0, "outputs" }, diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 00a06260ea..0e0da8f22b 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -5,8 +5,12 @@ #include <httpserver.h> #include <index/blockfilterindex.h> +#include <index/coinstatsindex.h> #include <index/txindex.h> #include <interfaces/chain.h> +#include <interfaces/echo.h> +#include <interfaces/init.h> +#include <interfaces/ipc.h> #include <key_io.h> #include <node/context.h> #include <outputtype.h> @@ -644,6 +648,43 @@ static RPCHelpMan echo(const std::string& name) static RPCHelpMan echo() { return echo("echo"); } static RPCHelpMan echojson() { return echo("echojson"); } +static RPCHelpMan echoipc() +{ + return RPCHelpMan{ + "echoipc", + "\nEcho back the input argument, passing it through a spawned process in a multiprocess build.\n" + "This command is for testing.\n", + {{"arg", RPCArg::Type::STR, RPCArg::Optional::NO, "The string to echo",}}, + RPCResult{RPCResult::Type::STR, "echo", "The echoed string."}, + RPCExamples{HelpExampleCli("echo", "\"Hello world\"") + + HelpExampleRpc("echo", "\"Hello world\"")}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + std::unique_ptr<interfaces::Echo> echo; + if (interfaces::Ipc* ipc = Assert(EnsureAnyNodeContext(request.context).init)->ipc()) { + // Spawn a new bitcoin-node process and call makeEcho to get a + // client pointer to a interfaces::Echo instance running in + // that process. This is just for testing. A slightly more + // realistic test spawning a different executable instead of + // the same executable would add a new bitcoin-echo executable, + // and spawn bitcoin-echo below instead of bitcoin-node. But + // using bitcoin-node avoids the need to build and install a + // new executable just for this one test. + auto init = ipc->spawnProcess("bitcoin-node"); + echo = init->makeEcho(); + ipc->addCleanup(*echo, [init = init.release()] { delete init; }); + } else { + // IPC support is not available because this is a bitcoind + // process not a bitcoind-node process, so just create a local + // interfaces::Echo object and return it so the `echoipc` RPC + // method will work, and the python test calling `echoipc` + // can expect the same result. + echo = interfaces::MakeEcho(); + } + return echo->echo(request.params[0].get_str()); + }, + }; +} + static UniValue SummaryToJSON(const IndexSummary&& summary, std::string index_name) { UniValue ret_summary(UniValue::VOBJ); @@ -689,6 +730,10 @@ static RPCHelpMan getindexinfo() result.pushKVs(SummaryToJSON(g_txindex->GetSummary(), index_name)); } + if (g_coin_stats_index) { + result.pushKVs(SummaryToJSON(g_coin_stats_index->GetSummary(), index_name)); + } + ForEachBlockFilterIndex([&result, &index_name](const BlockFilterIndex& index) { result.pushKVs(SummaryToJSON(index.GetSummary(), index_name)); }); @@ -719,6 +764,7 @@ static const CRPCCommand commands[] = { "hidden", &mockscheduler, }, { "hidden", &echo, }, { "hidden", &echojson, }, + { "hidden", &echoipc, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 2bfcaeafc9..382c6f11ff 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -277,7 +277,9 @@ static RPCHelpMan addnode() "\nAttempts to add or remove a node from the addnode list.\n" "Or try a connection to a node once.\n" "Nodes added using addnode (or -connect) are protected from DoS disconnection and are not required to be\n" - "full nodes/support SegWit as other outbound peers are (though such peers will not be synced from).\n", + "full nodes/support SegWit as other outbound peers are (though such peers will not be synced from).\n" + + strprintf("Addnode connections are limited to %u at a time", MAX_ADDNODE_CONNECTIONS) + + " and are counted separately from the -maxconnections limit.\n", { {"node", RPCArg::Type::STR, RPCArg::Optional::NO, "The node (see getpeerinfo for nodes)"}, {"command", RPCArg::Type::STR, RPCArg::Optional::NO, "'add' to add a node to the list, 'remove' to remove a node from the list, 'onetry' to try a connection to the node once"}, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 16ca3bb47d..186a77385a 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -397,7 +397,7 @@ static RPCHelpMan createrawtransaction() "For compatibility reasons, a dictionary, which holds the key-value pairs directly, is also\n" " accepted as second parameter.", { - {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED, "", { {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT}, }, diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index df3ee9f007..7cf25e0c82 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -74,12 +74,12 @@ void RPCTypeCheckObj(const UniValue& o, } } -CAmount AmountFromValue(const UniValue& value) +CAmount AmountFromValue(const UniValue& value, int decimals) { if (!value.isNum() && !value.isStr()) throw JSONRPCError(RPC_TYPE_ERROR, "Amount is not a number or string"); CAmount amount; - if (!ParseFixedPoint(value.getValStr(), 8, &amount)) + if (!ParseFixedPoint(value.getValStr(), decimals, &amount)) throw JSONRPCError(RPC_TYPE_ERROR, "Invalid amount"); if (!MoneyRange(amount)) throw JSONRPCError(RPC_TYPE_ERROR, "Amount out of range"); @@ -231,16 +231,12 @@ CTxDestination AddAndGetMultisigDestination(const int required, const std::vecto if ((int)pubkeys.size() < required) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("not enough keys supplied (got %u keys, but need at least %d to redeem)", pubkeys.size(), required)); } - if (pubkeys.size() > 16) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Number of keys involved in the multisignature address creation > 16\nReduce the number"); + if (pubkeys.size() > MAX_PUBKEYS_PER_MULTISIG) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Number of keys involved in the multisignature address creation > %d\nReduce the number", MAX_PUBKEYS_PER_MULTISIG)); } script_out = GetScriptForMultisig(required, pubkeys); - if (script_out.size() > MAX_SCRIPT_ELEMENT_SIZE) { - throw JSONRPCError(RPC_INVALID_PARAMETER, (strprintf("redeemScript exceeds size limit: %d > %d", script_out.size(), MAX_SCRIPT_ELEMENT_SIZE))); - } - // Check if any keys are uncompressed. If so, the type is legacy for (const CPubKey& pk : pubkeys) { if (!pk.IsCompressed()) { @@ -249,6 +245,10 @@ CTxDestination AddAndGetMultisigDestination(const int required, const std::vecto } } + if (type == OutputType::LEGACY && script_out.size() > MAX_SCRIPT_ELEMENT_SIZE) { + throw JSONRPCError(RPC_INVALID_PARAMETER, (strprintf("redeemScript exceeds size limit: %d > %d", script_out.size(), MAX_SCRIPT_ELEMENT_SIZE))); + } + // Make the address CTxDestination dest = AddAndGetDestinationForScript(keystore, script_out, type); diff --git a/src/rpc/util.h b/src/rpc/util.h index 8ec18b2f35..0120dbcc98 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -77,7 +77,14 @@ extern uint256 ParseHashO(const UniValue& o, std::string strKey); extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); -extern CAmount AmountFromValue(const UniValue& value); +/** + * Validate and return a CAmount from a UniValue number or string. + * + * @param[in] value UniValue number or string to parse. + * @param[in] decimals Number of significant digits (default: 8). + * @returns a CAmount if the various checks pass. + */ +extern CAmount AmountFromValue(const UniValue& value, int decimals = 8); using RPCArgList = std::vector<std::pair<std::string, UniValue>>; extern std::string HelpExampleCli(const std::string& methodname, const std::string& args); @@ -173,7 +180,7 @@ struct RPCArg { m_oneline_description{std::move(oneline_description)}, m_type_str{std::move(type_str)} { - CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ); + CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ && type != Type::OBJ_USER_KEYS); } RPCArg( @@ -193,7 +200,7 @@ struct RPCArg { m_oneline_description{std::move(oneline_description)}, m_type_str{std::move(type_str)} { - CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ); + CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ || type == Type::OBJ_USER_KEYS); } bool IsOptional() const; diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index f1433553bc..b54ba204f0 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -998,8 +998,8 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const providers.emplace_back(std::move(pk)); key_exp_index++; } - if (providers.empty() || providers.size() > 16) { - error = strprintf("Cannot have %u keys in multisig; must have between 1 and 16 keys, inclusive", providers.size()); + if (providers.empty() || providers.size() > MAX_PUBKEYS_PER_MULTISIG) { + error = strprintf("Cannot have %u keys in multisig; must have between 1 and %d keys, inclusive", providers.size(), MAX_PUBKEYS_PER_MULTISIG); return nullptr; } else if (thres < 1) { error = strprintf("Multisig threshold cannot be %d, must be at least 1", thres); @@ -1015,6 +1015,7 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const } } if (ctx == ParseScriptContext::P2SH) { + // This limits the maximum number of compressed pubkeys to 15. if (script_size + 3 > MAX_SCRIPT_ELEMENT_SIZE) { error = strprintf("P2SH script is too large, %d bytes is larger than %d bytes", script_size + 3, MAX_SCRIPT_ELEMENT_SIZE); return nullptr; @@ -1097,7 +1098,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo TxoutType txntype = Solver(script, data); if (txntype == TxoutType::PUBKEY) { - CPubKey pubkey(data[0].begin(), data[0].end()); + CPubKey pubkey(data[0]); if (pubkey.IsValid()) { return std::make_unique<PKDescriptor>(InferPubkey(pubkey, ctx, provider)); } @@ -1121,7 +1122,7 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo if (txntype == TxoutType::MULTISIG) { std::vector<std::unique_ptr<PubkeyProvider>> providers; for (size_t i = 1; i + 1 < data.size(); ++i) { - CPubKey pubkey(data[i].begin(), data[i].end()); + CPubKey pubkey(data[i]); providers.push_back(InferPubkey(pubkey, ctx, provider)); } return std::make_unique<MultisigDescriptor>((int)data[0][0], std::move(providers)); diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index abc0625bb1..dc0f165be0 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -225,7 +225,7 @@ bool static CheckPubKeyEncoding(const valtype &vchPubKey, unsigned int flags, co return true; } -bool static CheckMinimalPush(const valtype& data, opcodetype opcode) { +bool CheckMinimalPush(const valtype& data, opcodetype opcode) { // Excludes OP_1NEGATE, OP_1-16 since they are by definition minimal assert(0 <= opcode && opcode <= OP_PUSHDATA4); if (data.size() == 0) { @@ -1890,7 +1890,7 @@ static bool VerifyWitnessProgram(const CScriptWitness& witness, int witversion, const valtype& script_bytes = SpanPopBack(stack); exec_script = CScript(script_bytes.begin(), script_bytes.end()); uint256 hash_exec_script; - CSHA256().Write(&exec_script[0], exec_script.size()).Finalize(hash_exec_script.begin()); + CSHA256().Write(exec_script.data(), exec_script.size()).Finalize(hash_exec_script.begin()); if (memcmp(hash_exec_script.begin(), program.data(), 32)) { return set_error(serror, SCRIPT_ERR_WITNESS_PROGRAM_MISMATCH); } diff --git a/src/script/interpreter.h b/src/script/interpreter.h index c76b3acb22..212de17c7b 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -316,6 +316,8 @@ bool VerifyScript(const CScript& scriptSig, const CScript& scriptPubKey, const C size_t CountWitnessSigOps(const CScript& scriptSig, const CScript& scriptPubKey, const CScriptWitness* witness, unsigned int flags); +bool CheckMinimalPush(const std::vector<unsigned char>& data, opcodetype opcode); + int FindAndDelete(CScript& script, const CScript& b); #endif // BITCOIN_SCRIPT_INTERPRETER_H diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index c6d898a25a..65867c1c14 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -53,14 +53,14 @@ public: ComputeEntryECDSA(uint256& entry, const uint256 &hash, const std::vector<unsigned char>& vchSig, const CPubKey& pubkey) const { CSHA256 hasher = m_salted_hasher_ecdsa; - hasher.Write(hash.begin(), 32).Write(&pubkey[0], pubkey.size()).Write(&vchSig[0], vchSig.size()).Finalize(entry.begin()); + hasher.Write(hash.begin(), 32).Write(pubkey.data(), pubkey.size()).Write(vchSig.data(), vchSig.size()).Finalize(entry.begin()); } void ComputeEntrySchnorr(uint256& entry, const uint256 &hash, Span<const unsigned char> sig, const XOnlyPubKey& pubkey) const { CSHA256 hasher = m_salted_hasher_schnorr; - hasher.Write(hash.begin(), 32).Write(&pubkey[0], pubkey.size()).Write(sig.data(), sig.size()).Finalize(entry.begin()); + hasher.Write(hash.begin(), 32).Write(pubkey.data(), pubkey.size()).Write(sig.data(), sig.size()).Finalize(entry.begin()); } bool diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 4d9026427e..da0092f9e3 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -167,7 +167,7 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator return true; case TxoutType::WITNESS_V0_SCRIPTHASH: - CRIPEMD160().Write(&vSolutions[0][0], vSolutions[0].size()).Finalize(h160.begin()); + CRIPEMD160().Write(vSolutions[0].data(), vSolutions[0].size()).Finalize(h160.begin()); if (GetCScript(provider, sigdata, CScriptID{h160}, scriptRet)) { ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); return true; diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 700155c8d4..364fac3c84 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -88,21 +88,53 @@ static constexpr bool IsSmallInteger(opcodetype opcode) return opcode >= OP_1 && opcode <= OP_16; } -static bool MatchMultisig(const CScript& script, unsigned int& required, std::vector<valtype>& pubkeys) +static constexpr bool IsPushdataOp(opcodetype opcode) +{ + return opcode > OP_FALSE && opcode <= OP_PUSHDATA4; +} + +static constexpr bool IsValidMultisigKeyCount(int n_keys) +{ + return n_keys > 0 && n_keys <= MAX_PUBKEYS_PER_MULTISIG; +} + +static bool GetMultisigKeyCount(opcodetype opcode, valtype data, int& count) +{ + if (IsSmallInteger(opcode)) { + count = CScript::DecodeOP_N(opcode); + return IsValidMultisigKeyCount(count); + } + + if (IsPushdataOp(opcode)) { + if (!CheckMinimalPush(data, opcode)) return false; + try { + count = CScriptNum(data, /* fRequireMinimal = */ true).getint(); + return IsValidMultisigKeyCount(count); + } catch (const scriptnum_error&) { + return false; + } + } + + return false; +} + +static bool MatchMultisig(const CScript& script, int& required_sigs, std::vector<valtype>& pubkeys) { opcodetype opcode; valtype data; + int num_keys; + CScript::const_iterator it = script.begin(); if (script.size() < 1 || script.back() != OP_CHECKMULTISIG) return false; - if (!script.GetOp(it, opcode, data) || !IsSmallInteger(opcode)) return false; - required = CScript::DecodeOP_N(opcode); + if (!script.GetOp(it, opcode, data) || !GetMultisigKeyCount(opcode, data, required_sigs)) return false; while (script.GetOp(it, opcode, data) && CPubKey::ValidSize(data)) { pubkeys.emplace_back(std::move(data)); } - if (!IsSmallInteger(opcode)) return false; - unsigned int keys = CScript::DecodeOP_N(opcode); - if (pubkeys.size() != keys || keys < required) return false; + if (!GetMultisigKeyCount(opcode, data, num_keys)) return false; + + if (pubkeys.size() != static_cast<unsigned long>(num_keys) || num_keys < required_sigs) return false; + return (it + 1 == script.end()); } @@ -163,12 +195,12 @@ TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned c return TxoutType::PUBKEYHASH; } - unsigned int required; + int required; std::vector<std::vector<unsigned char>> keys; if (MatchMultisig(scriptPubKey, required, keys)) { - vSolutionsRet.push_back({static_cast<unsigned char>(required)}); // safe as required is in range 1..16 + vSolutionsRet.push_back({static_cast<unsigned char>(required)}); // safe as required is in range 1..20 vSolutionsRet.insert(vSolutionsRet.end(), keys.begin(), keys.end()); - vSolutionsRet.push_back({static_cast<unsigned char>(keys.size())}); // safe as size is in range 1..16 + vSolutionsRet.push_back({static_cast<unsigned char>(keys.size())}); // safe as size is in range 1..20 return TxoutType::MULTISIG; } @@ -318,10 +350,11 @@ CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys) { CScript script; - script << CScript::EncodeOP_N(nRequired); + script << nRequired; for (const CPubKey& key : keys) script << ToByteVector(key); - script << CScript::EncodeOP_N(keys.size()) << OP_CHECKMULTISIG; + script << keys.size() << OP_CHECKMULTISIG; + return script; } diff --git a/src/streams.h b/src/streams.h index e78da31cbc..31407287ae 100644 --- a/src/streams.h +++ b/src/streams.h @@ -167,7 +167,7 @@ public: } template<typename T> - VectorReader& operator>>(T& obj) + VectorReader& operator>>(T&& obj) { // Unserialize from this stream ::Unserialize(*this, obj); diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp new file mode 100644 index 0000000000..3fc7b72077 --- /dev/null +++ b/src/test/coinstatsindex_tests.cpp @@ -0,0 +1,79 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <index/coinstatsindex.h> +#include <test/util/setup_common.h> +#include <util/time.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> + +#include <chrono> + + +BOOST_AUTO_TEST_SUITE(coinstatsindex_tests) + +BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) +{ + CoinStatsIndex coin_stats_index{1 << 20, true}; + + CCoinsStats coin_stats{CoinStatsHashType::MUHASH}; + const CBlockIndex* block_index; + { + LOCK(cs_main); + block_index = ChainActive().Tip(); + } + + // CoinStatsIndex should not be found before it is started. + BOOST_CHECK(!coin_stats_index.LookUpStats(block_index, coin_stats)); + + // BlockUntilSyncedToCurrentChain should return false before CoinStatsIndex + // is started. + BOOST_CHECK(!coin_stats_index.BlockUntilSyncedToCurrentChain()); + + coin_stats_index.Start(); + + // Allow the CoinStatsIndex to catch up with the block index that is syncing + // in a background thread. + const auto timeout = GetTime<std::chrono::seconds>() + 120s; + while (!coin_stats_index.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(timeout > GetTime<std::chrono::milliseconds>()); + UninterruptibleSleep(100ms); + } + + // Check that CoinStatsIndex works for genesis block. + const CBlockIndex* genesis_block_index; + { + LOCK(cs_main); + genesis_block_index = ChainActive().Genesis(); + } + BOOST_CHECK(coin_stats_index.LookUpStats(genesis_block_index, coin_stats)); + + // Check that CoinStatsIndex updates with new blocks. + coin_stats_index.LookUpStats(block_index, coin_stats); + + const CScript script_pub_key{CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG}; + std::vector<CMutableTransaction> noTxns; + CreateAndProcessBlock(noTxns, script_pub_key); + + // Let the CoinStatsIndex to catch up again. + BOOST_CHECK(coin_stats_index.BlockUntilSyncedToCurrentChain()); + + CCoinsStats new_coin_stats{CoinStatsHashType::MUHASH}; + const CBlockIndex* new_block_index; + { + LOCK(cs_main); + new_block_index = ChainActive().Tip(); + } + coin_stats_index.LookUpStats(new_block_index, new_coin_stats); + + BOOST_CHECK(block_index != new_block_index); + + // Shutdown sequence (c.f. Shutdown() in init.cpp) + coin_stats_index.Stop(); + + // Rest of shutdown sequence and destructors happen in ~TestingSetup() +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 4ddbc8338e..b7ba46bd30 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -72,14 +72,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_ckey_id) CScript script = CScript() << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; BOOST_CHECK_EQUAL(script.size(), 25U); - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script BOOST_CHECK_EQUAL(out.size(), 21U); BOOST_CHECK_EQUAL(out[0], 0x00); - BOOST_CHECK_EQUAL(memcmp(&out[1], &script[3], 20), 0); // compare the 20 relevant chars of the CKeyId in the script + BOOST_CHECK_EQUAL(memcmp(out.data() + 1, script.data() + 3, 20), 0); // compare the 20 relevant chars of the CKeyId in the script } BOOST_AUTO_TEST_CASE(compress_script_to_cscript_id) @@ -89,14 +89,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_cscript_id) script << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; BOOST_CHECK_EQUAL(script.size(), 23U); - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script BOOST_CHECK_EQUAL(out.size(), 21U); BOOST_CHECK_EQUAL(out[0], 0x01); - BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 20), 0); // compare the 20 relevant chars of the CScriptId in the script + BOOST_CHECK_EQUAL(memcmp(out.data() + 1, script.data() + 2, 20), 0); // compare the 20 relevant chars of the CScriptId in the script } BOOST_AUTO_TEST_CASE(compress_script_to_compressed_pubkey_id) @@ -107,14 +107,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_compressed_pubkey_id) CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // COMPRESSED_PUBLIC_KEY_SIZE (33) BOOST_CHECK_EQUAL(script.size(), 35U); - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script BOOST_CHECK_EQUAL(out.size(), 33U); - BOOST_CHECK_EQUAL(memcmp(&out[0], &script[1], 1), 0); - BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 32), 0); // compare the 32 chars of the compressed CPubKey + BOOST_CHECK_EQUAL(memcmp(out.data(), script.data() + 1, 1), 0); + BOOST_CHECK_EQUAL(memcmp(out.data() + 1, script.data() + 2, 32), 0); // compare the 32 chars of the compressed CPubKey } BOOST_AUTO_TEST_CASE(compress_script_to_uncompressed_pubkey_id) @@ -124,13 +124,13 @@ BOOST_AUTO_TEST_CASE(compress_script_to_uncompressed_pubkey_id) CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // PUBLIC_KEY_SIZE (65) BOOST_CHECK_EQUAL(script.size(), 67U); // 1 char code + 65 char pubkey + OP_CHECKSIG - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script BOOST_CHECK_EQUAL(out.size(), 33U); - BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 32), 0); // first 32 chars of CPubKey are copied into out[1:] + BOOST_CHECK_EQUAL(memcmp(out.data() + 1, script.data() + 2, 32), 0); // first 32 chars of CPubKey are copied into out[1:] BOOST_CHECK_EQUAL(out[0], 0x04 | (script[65] & 0x01)); // least significant bit (lsb) of last char of pubkey is mapped into out[0] } diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 7358b246b6..edec5f0a31 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -33,7 +33,7 @@ static void TestVector(const Hasher &h, const In &in, const Out &out) { hash.resize(out.size()); { // Test that writing the whole input string at once works. - Hasher(h).Write((unsigned char*)&in[0], in.size()).Finalize(&hash[0]); + Hasher(h).Write((const uint8_t*)in.data(), in.size()).Finalize(hash.data()); BOOST_CHECK(hash == out); } for (int i=0; i<32; i++) { @@ -42,15 +42,15 @@ static void TestVector(const Hasher &h, const In &in, const Out &out) { size_t pos = 0; while (pos < in.size()) { size_t len = InsecureRandRange((in.size() - pos + 1) / 2 + 1); - hasher.Write((unsigned char*)&in[pos], len); + hasher.Write((const uint8_t*)in.data() + pos, len); pos += len; if (pos > 0 && pos + 2 * out.size() > in.size() && pos < in.size()) { // Test that writing the rest at once to a copy of a hasher works. - Hasher(hasher).Write((unsigned char*)&in[pos], in.size() - pos).Finalize(&hash[0]); + Hasher(hasher).Write((const uint8_t*)in.data() + pos, in.size() - pos).Finalize(hash.data()); BOOST_CHECK(hash == out); } } - hasher.Finalize(&hash[0]); + hasher.Finalize(hash.data()); BOOST_CHECK(hash == out); } } diff --git a/src/test/data/tx_invalid.json b/src/test/data/tx_invalid.json index 41bebab2a0..a47bc8f366 100644 --- a/src/test/data/tx_invalid.json +++ b/src/test/data/tx_invalid.json @@ -311,6 +311,10 @@ [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x00 0x20 0x34b6c399093e06cf9f0f7f660a1abcfe78fcf7b576f43993208edd9518a0ae9b", 1000]], "0100000000010100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015101045102010100000000", "P2SH,WITNESS"], +["P2WSH with an empty redeem should fail due to empty stack"], +[[["3d4da21b04a67a54c8a58df1c53a0534b0a7f0864fb3d19abd43b8f6934e785f", 0, "0x00 0x20 0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 1337]], +"020000000001015f784e93f6b843bd9ad1b34f86f0a7b034053ac5f18da5c8547aa6041ba24d3d0000000000ffffffff013905000000000000220020e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855010000000000", "P2SH,WITNESS"], + ["33 bytes push should be considered a witness scriptPubKey"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "0x60 0x21 0xff25429251b5a84f452230a3c75fd886b7fc5a7865ce4a7bb7a9d7c5be6da3dbff", 1000]], "010000000100010000000000000000000000000000000000000000000000000000000000000000000000ffffffff01e803000000000000015100000000", "P2SH,WITNESS,DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM"], diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index ea41a03728..36e2dac3ff 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <pubkey.h> #include <script/descriptor.h> #include <script/sign.h> #include <script/standard.h> @@ -27,6 +28,14 @@ void CheckUnparsable(const std::string& prv, const std::string& pub, const std:: BOOST_CHECK_EQUAL(error, expected_error); } +/** Check that the script is inferred as non-standard */ +void CheckInferRaw(const CScript& script) +{ + FlatSigningProvider dummy_provider; + std::unique_ptr<Descriptor> desc = InferDescriptor(script, dummy_provider); + BOOST_CHECK(desc->ToString().rfind("raw(", 0) == 0); +} + constexpr int DEFAULT = 0; constexpr int RANGE = 1; // Expected to be ranged descriptor constexpr int HARDENED = 2; // Derivation needs access to private keys @@ -351,8 +360,9 @@ BOOST_AUTO_TEST_CASE(descriptor_test) CheckUnparsable("multi(0,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(0,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multisig threshold cannot be 0, must be at least 1"); // Threshold of 0 CheckUnparsable("multi(3,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(3,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multisig threshold cannot be larger than the number of keys; threshold is 3 but only 2 keys specified"); // Threshold larger than number of keys CheckUnparsable("multi(3,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f)", "multi(3,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8)", "Cannot have 4 pubkeys in bare multisig; only at most 3 pubkeys"); // Threshold larger than number of keys - CheckUnparsable("sh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))","sh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "Cannot have 17 keys in multisig; must have between 1 and 16 keys, inclusive"); // Cannot have more than 16 keys in a multisig - + CheckUnparsable("sh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))","sh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "P2SH script is too large, 581 bytes is larger than 520 bytes"); // Cannot have more than 15 keys in a P2SH multisig, or we exceed maximum push size + Check("wsh(multi(20,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9,KzRedjSwMggebB3VufhbzpYJnvHfHe9kPJSjCU5QpJdAW3NSZxYS,Kyjtp5858xL7JfeV4PNRCKy2t6XvgqNNepArGY9F9F1SSPqNEMs3,L2D4RLHPiHBidkHS8ftx11jJk1hGFELvxh8LoxNQheaGT58dKenW,KyLPZdwY4td98bKkXqEXTEBX3vwEYTQo1yyLjX2jKXA63GBpmSjv))","wsh(multi(20,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232,02bc2feaa536991d269aae46abb8f3772a5b3ad592314945e51543e7da84c4af6e,0318bf32e5217c1eb771a6d5ce1cd39395dff7ff665704f175c9a5451d95a2f2ca,02c681a6243f16208c2004bb81f5a8a67edfdd3e3711534eadeec3dcf0b010c759,0249fdd6b69768b8d84b4893f8ff84b36835c50183de20fcae8f366a45290d01fd))", "wsh(multi(20,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9,KzRedjSwMggebB3VufhbzpYJnvHfHe9kPJSjCU5QpJdAW3NSZxYS,Kyjtp5858xL7JfeV4PNRCKy2t6XvgqNNepArGY9F9F1SSPqNEMs3,L2D4RLHPiHBidkHS8ftx11jJk1hGFELvxh8LoxNQheaGT58dKenW,KyLPZdwY4td98bKkXqEXTEBX3vwEYTQo1yyLjX2jKXA63GBpmSjv))","wsh(multi(20,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232,02bc2feaa536991d269aae46abb8f3772a5b3ad592314945e51543e7da84c4af6e,0318bf32e5217c1eb771a6d5ce1cd39395dff7ff665704f175c9a5451d95a2f2ca,02c681a6243f16208c2004bb81f5a8a67edfdd3e3711534eadeec3dcf0b010c759,0249fdd6b69768b8d84b4893f8ff84b36835c50183de20fcae8f366a45290d01fd))", SIGNABLE, {{"0020376bd8344b8b6ebe504ff85ef743eaa1aa9272178223bcb6887e9378efb341ac"}}, OutputType::BECH32); // In P2WSH we can have up to 20 keys +Check("sh(wsh(multi(20,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9,KzRedjSwMggebB3VufhbzpYJnvHfHe9kPJSjCU5QpJdAW3NSZxYS,Kyjtp5858xL7JfeV4PNRCKy2t6XvgqNNepArGY9F9F1SSPqNEMs3,L2D4RLHPiHBidkHS8ftx11jJk1hGFELvxh8LoxNQheaGT58dKenW,KyLPZdwY4td98bKkXqEXTEBX3vwEYTQo1yyLjX2jKXA63GBpmSjv)))","sh(wsh(multi(20,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232,02bc2feaa536991d269aae46abb8f3772a5b3ad592314945e51543e7da84c4af6e,0318bf32e5217c1eb771a6d5ce1cd39395dff7ff665704f175c9a5451d95a2f2ca,02c681a6243f16208c2004bb81f5a8a67edfdd3e3711534eadeec3dcf0b010c759,0249fdd6b69768b8d84b4893f8ff84b36835c50183de20fcae8f366a45290d01fd)))", "sh(wsh(multi(20,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9,KzRedjSwMggebB3VufhbzpYJnvHfHe9kPJSjCU5QpJdAW3NSZxYS,Kyjtp5858xL7JfeV4PNRCKy2t6XvgqNNepArGY9F9F1SSPqNEMs3,L2D4RLHPiHBidkHS8ftx11jJk1hGFELvxh8LoxNQheaGT58dKenW,KyLPZdwY4td98bKkXqEXTEBX3vwEYTQo1yyLjX2jKXA63GBpmSjv)))","sh(wsh(multi(20,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232,02bc2feaa536991d269aae46abb8f3772a5b3ad592314945e51543e7da84c4af6e,0318bf32e5217c1eb771a6d5ce1cd39395dff7ff665704f175c9a5451d95a2f2ca,02c681a6243f16208c2004bb81f5a8a67edfdd3e3711534eadeec3dcf0b010c759,0249fdd6b69768b8d84b4893f8ff84b36835c50183de20fcae8f366a45290d01fd)))", SIGNABLE, {{"a914c2c9c510e9d7f92fd6131e94803a8d34a8ef675e87"}}, OutputType::P2SH_SEGWIT); // Even if it's wrapped into P2SH // Check for invalid nesting of structures CheckUnparsable("sh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "sh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "A function is needed within P2SH"); // P2SH needs a script, not a key CheckUnparsable("sh(combo(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(combo(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "Can only have combo() at top level"); // Old must be top level @@ -376,6 +386,27 @@ BOOST_AUTO_TEST_CASE(descriptor_test) CheckUnparsable("", "addr(asdf)", "Address is not valid"); // Invalid address CheckUnparsable("", "raw(asdf)", "Raw script is not hex"); // Invalid script CheckUnparsable("", "raw(Ü)#00000000", "Invalid characters in payload"); // Invalid chars + + // A 2of4 but using a direct push rather than OP_2 + CScript nonminimalmultisig; + CKey keys[4]; + nonminimalmultisig << std::vector<unsigned char>{2}; + for (int i = 0; i < 4; i++) { + keys[i].MakeNewKey(true); + nonminimalmultisig << ToByteVector(keys[i].GetPubKey()); + } + nonminimalmultisig << 4 << OP_CHECKMULTISIG; + CheckInferRaw(nonminimalmultisig); + + // A 2of4 but using a direct push rather than OP_4 + nonminimalmultisig.clear(); + nonminimalmultisig << 2; + for (int i = 0; i < 4; i++) { + keys[i].MakeNewKey(true); + nonminimalmultisig << ToByteVector(keys[i].GetPubKey()); + } + nonminimalmultisig << std::vector<unsigned char>{4} << OP_CHECKMULTISIG; + CheckInferRaw(nonminimalmultisig); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index b21d2eae79..21dc80cc8d 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -258,10 +258,10 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) (void)GetTransactionSigOpCost(transaction, coins_view_cache, flags); }, [&] { - CCoinsStats stats; + CCoinsStats stats{CoinStatsHashType::HASH_SERIALIZED}; bool expected_code_path = false; try { - (void)GetUTXOStats(&coins_view_cache, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), stats, CoinStatsHashType::HASH_SERIALIZED); + (void)GetUTXOStats(&coins_view_cache, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), stats); } catch (const std::logic_error&) { expected_code_path = true; } diff --git a/src/test/fuzz/danger_link_all.sh b/src/test/fuzz/danger_link_all.sh deleted file mode 100755 index 2ddd00c658..0000000000 --- a/src/test/fuzz/danger_link_all.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -set -e - -ROOT_DIR="$(git rev-parse --show-toplevel)" - -# Run only once (break make recursion) -if [ -d "${ROOT_DIR}/lock_fuzz_link_all" ]; then - exit -fi -mkdir "${ROOT_DIR}/lock_fuzz_link_all" - -echo "Linking each fuzz target separately." -for FUZZING_HARNESS in $(PRINT_ALL_FUZZ_TARGETS_AND_ABORT=1 "${ROOT_DIR}/src/test/fuzz/fuzz" | sort -u); do - echo "Building src/test/fuzz/${FUZZING_HARNESS} ..." - git checkout -- "${ROOT_DIR}/src/test/fuzz/fuzz.cpp" - sed -i "s/std::getenv(\"FUZZ\")/\"${FUZZING_HARNESS}\"/g" "${ROOT_DIR}/src/test/fuzz/fuzz.cpp" - make - mv "${ROOT_DIR}/src/test/fuzz/fuzz" "${ROOT_DIR}/src/test/fuzz/${FUZZING_HARNESS}" -done -git checkout -- "${ROOT_DIR}/src/test/fuzz/fuzz.cpp" -rmdir "${ROOT_DIR}/lock_fuzz_link_all" -echo "Successfully built all fuzz targets." diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 1fab46ff13..0d8d960d56 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -29,13 +29,24 @@ static TypeTestOneInput* g_test_one_input{nullptr}; void initialize() { + bool should_abort{false}; if (std::getenv("PRINT_ALL_FUZZ_TARGETS_AND_ABORT")) { for (const auto& t : FuzzTargets()) { if (std::get<2>(t.second)) continue; std::cout << t.first << std::endl; } - Assert(false); + should_abort = true; } + if (const char* out_path = std::getenv("WRITE_ALL_FUZZ_TARGETS_AND_ABORT")) { + std::cout << "Writing all fuzz target names to '" << out_path << "'." << std::endl; + std::ofstream out_stream(out_path, std::ios::binary); + for (const auto& t : FuzzTargets()) { + if (std::get<2>(t.second)) continue; + out_stream << t.first << std::endl; + } + should_abort = true; + } + Assert(!should_abort); std::string_view fuzz_target{Assert(std::getenv("FUZZ"))}; const auto it = FuzzTargets().find(fuzz_target); Assert(it != FuzzTargets().end()); diff --git a/src/test/fuzz/parse_iso8601.cpp b/src/test/fuzz/parse_iso8601.cpp index dcb24ac127..a56f2aa48d 100644 --- a/src/test/fuzz/parse_iso8601.cpp +++ b/src/test/fuzz/parse_iso8601.cpp @@ -15,7 +15,7 @@ FUZZ_TARGET(parse_iso8601) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const int64_t random_time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const int64_t random_time = fuzzed_data_provider.ConsumeIntegral<int32_t>(); const std::string random_string = fuzzed_data_provider.ConsumeRemainingBytesAsString(); const std::string iso8601_datetime = FormatISO8601DateTime(random_time); diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 96e1cfa08f..7b99193ad0 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -100,7 +100,6 @@ void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE g_setup->m_node.peerman->SendMessages(&p2p_node); } SyncWithValidationInterfaceQueue(); - LOCK2(::cs_main, g_cs_orphans); // See init.cpp for rationale for implicit locking order requirement g_setup->m_node.connman->StopNodes(); } diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 203c0ef8e1..11b236c9bd 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -80,6 +80,5 @@ FUZZ_TARGET_INIT(process_messages, initialize_process_messages) } } SyncWithValidationInterfaceQueue(); - LOCK2(::cs_main, g_cs_orphans); // See init.cpp for rationale for implicit locking order requirement g_setup->m_node.connman->StopNodes(); } diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp new file mode 100644 index 0000000000..cf32a79932 --- /dev/null +++ b/src/test/fuzz/rpc.cpp @@ -0,0 +1,358 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <base58.h> +#include <core_io.h> +#include <key.h> +#include <key_io.h> +#include <node/context.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <psbt.h> +#include <rpc/blockchain.h> +#include <rpc/client.h> +#include <rpc/request.h> +#include <rpc/server.h> +#include <rpc/util.h> +#include <span.h> +#include <streams.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <tinyformat.h> +#include <univalue.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/time.h> + +#include <cstdint> +#include <iostream> +#include <memory> +#include <optional> +#include <stdexcept> +#include <string> +#include <vector> + +namespace { +struct RPCFuzzTestingSetup : public TestingSetup { + RPCFuzzTestingSetup(const std::string& chain_name, const std::vector<const char*>& extra_args) : TestingSetup{chain_name, extra_args} + { + } + + UniValue CallRPC(const std::string& rpc_method, const std::vector<std::string>& arguments) + { + JSONRPCRequest request; + request.context = &m_node; + request.strMethod = rpc_method; + request.params = RPCConvertValues(rpc_method, arguments); + return tableRPC.execute(request); + } + + std::vector<std::string> GetRPCCommands() const + { + return tableRPC.listCommands(); + } +}; + +RPCFuzzTestingSetup* rpc_testing_setup = nullptr; +std::string g_limit_to_rpc_command; + +// RPC commands which are not appropriate for fuzzing: such as RPC commands +// reading or writing to a filename passed as an RPC parameter, RPC commands +// resulting in network activity, etc. +const std::vector<std::string> RPC_COMMANDS_NOT_SAFE_FOR_FUZZING{ + "addconnection", // avoid DNS lookups + "addnode", // avoid DNS lookups + "addpeeraddress", // avoid DNS lookups + "analyzepsbt", // avoid signed integer overflow in CFeeRate::GetFee(unsigned long) (https://github.com/bitcoin/bitcoin/issues/20607) + "dumptxoutset", // avoid writing to disk + "dumpwallet", // avoid writing to disk + "echoipc", // avoid assertion failure (Assertion `"EnsureAnyNodeContext(request.context).init" && check' failed.) + "generatetoaddress", // avoid prohibitively slow execution (when `num_blocks` is large) + "generatetodescriptor", // avoid prohibitively slow execution (when `nblocks` is large) + "gettxoutproof", // avoid prohibitively slow execution + "importwallet", // avoid reading from disk + "loadwallet", // avoid reading from disk + "prioritisetransaction", // avoid signed integer overflow in CTxMemPool::PrioritiseTransaction(uint256 const&, long const&) (https://github.com/bitcoin/bitcoin/issues/20626) + "savemempool", // disabled as a precautionary measure: may take a file path argument in the future + "setban", // avoid DNS lookups + "stop", // avoid shutdown state +}; + +// RPC commands which are safe for fuzzing. +const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ + "clearbanned", + "combinepsbt", + "combinerawtransaction", + "converttopsbt", + "createmultisig", + "createpsbt", + "createrawtransaction", + "decodepsbt", + "decoderawtransaction", + "decodescript", + "deriveaddresses", + "disconnectnode", + "echo", + "echojson", + "estimaterawfee", + "estimatesmartfee", + "finalizepsbt", + "generate", + "generateblock", + "getaddednodeinfo", + "getbestblockhash", + "getblock", + "getblockchaininfo", + "getblockcount", + "getblockfilter", + "getblockhash", + "getblockheader", + "getblockstats", + "getblocktemplate", + "getchaintips", + "getchaintxstats", + "getconnectioncount", + "getdescriptorinfo", + "getdifficulty", + "getindexinfo", + "getmemoryinfo", + "getmempoolancestors", + "getmempooldescendants", + "getmempoolentry", + "getmempoolinfo", + "getmininginfo", + "getnettotals", + "getnetworkhashps", + "getnetworkinfo", + "getnodeaddresses", + "getpeerinfo", + "getrawmempool", + "getrawtransaction", + "getrpcinfo", + "gettxout", + "gettxoutsetinfo", + "help", + "invalidateblock", + "joinpsbts", + "listbanned", + "logging", + "mockscheduler", + "ping", + "preciousblock", + "pruneblockchain", + "reconsiderblock", + "scantxoutset", + "sendrawtransaction", + "setmocktime", + "setnetworkactive", + "signmessagewithprivkey", + "signrawtransactionwithkey", + "submitblock", + "submitheader", + "syncwithvalidationinterfacequeue", + "testmempoolaccept", + "uptime", + "utxoupdatepsbt", + "validateaddress", + "verifychain", + "verifymessage", + "verifytxoutproof", + "waitforblock", + "waitforblockheight", + "waitfornewblock", +}; + +std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +{ + const size_t max_string_length = 4096; + std::string r; + CallOneOf( + fuzzed_data_provider, + [&] { + // string argument + r = fuzzed_data_provider.ConsumeRandomLengthString(max_string_length); + }, + [&] { + // base64 argument + r = EncodeBase64(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length)); + }, + [&] { + // hex argument + r = HexStr(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length)); + }, + [&] { + // bool argument + r = fuzzed_data_provider.ConsumeBool() ? "true" : "false"; + }, + [&] { + // range argument + r = "[" + ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()) + "," + ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()) + "]"; + }, + [&] { + // integral argument (int64_t) + r = ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()); + }, + [&] { + // integral argument (uint64_t) + r = ToString(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + }, + [&] { + // floating point argument + r = strprintf("%f", fuzzed_data_provider.ConsumeFloatingPoint<double>()); + }, + [&] { + // tx destination argument + r = EncodeDestination(ConsumeTxDestination(fuzzed_data_provider)); + }, + [&] { + // uint160 argument + r = ConsumeUInt160(fuzzed_data_provider).ToString(); + }, + [&] { + // uint256 argument + r = ConsumeUInt256(fuzzed_data_provider).ToString(); + }, + [&] { + // base32 argument + r = EncodeBase32(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length)); + }, + [&] { + // base58 argument + r = EncodeBase58(MakeUCharSpan(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length))); + }, + [&] { + // base58 argument with checksum + r = EncodeBase58Check(MakeUCharSpan(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length))); + }, + [&] { + // hex encoded block + std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + if (!opt_block) { + return; + } + CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + data_stream << *opt_block; + r = HexStr(data_stream); + }, + [&] { + // hex encoded block header + std::optional<CBlockHeader> opt_block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); + if (!opt_block_header) { + return; + } + CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + data_stream << *opt_block_header; + r = HexStr(data_stream); + }, + [&] { + // hex encoded tx + std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!opt_tx) { + return; + } + CDataStream data_stream{SER_NETWORK, fuzzed_data_provider.ConsumeBool() ? PROTOCOL_VERSION : (PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)}; + data_stream << *opt_tx; + r = HexStr(data_stream); + }, + [&] { + // base64 encoded psbt + std::optional<PartiallySignedTransaction> opt_psbt = ConsumeDeserializable<PartiallySignedTransaction>(fuzzed_data_provider); + if (!opt_psbt) { + return; + } + CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + data_stream << *opt_psbt; + r = EncodeBase64({data_stream.begin(), data_stream.end()}); + }, + [&] { + // base58 encoded key + const std::vector<uint8_t> random_bytes = fuzzed_data_provider.ConsumeBytes<uint8_t>(32); + CKey key; + key.Set(random_bytes.begin(), random_bytes.end(), fuzzed_data_provider.ConsumeBool()); + if (!key.IsValid()) { + return; + } + r = EncodeSecret(key); + }, + [&] { + // hex encoded pubkey + const std::vector<uint8_t> random_bytes = fuzzed_data_provider.ConsumeBytes<uint8_t>(32); + CKey key; + key.Set(random_bytes.begin(), random_bytes.end(), fuzzed_data_provider.ConsumeBool()); + if (!key.IsValid()) { + return; + } + r = HexStr(key.GetPubKey()); + }); + return r; +} + +std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +{ + std::vector<std::string> scalar_arguments; + while (fuzzed_data_provider.ConsumeBool()) { + scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider)); + } + return "[\"" + Join(scalar_arguments, "\",\"") + "\"]"; +} + +std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +{ + return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider) : ConsumeArrayRPCArgument(fuzzed_data_provider); +} + +RPCFuzzTestingSetup* InitializeRPCFuzzTestingSetup() +{ + static const auto setup = MakeNoLogFileContext<RPCFuzzTestingSetup>(); + SetRPCWarmupFinished(); + return setup.get(); +} +}; // namespace + +void initialize_rpc() +{ + rpc_testing_setup = InitializeRPCFuzzTestingSetup(); + const std::vector<std::string> supported_rpc_commands = rpc_testing_setup->GetRPCCommands(); + for (const std::string& rpc_command : supported_rpc_commands) { + const bool safe_for_fuzzing = std::find(RPC_COMMANDS_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_SAFE_FOR_FUZZING.end(); + const bool not_safe_for_fuzzing = std::find(RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.end(); + if (!(safe_for_fuzzing || not_safe_for_fuzzing)) { + std::cerr << "Error: RPC command \"" << rpc_command << "\" not found in RPC_COMMANDS_SAFE_FOR_FUZZING or RPC_COMMANDS_NOT_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n"; + std::terminate(); + } + if (safe_for_fuzzing && not_safe_for_fuzzing) { + std::cerr << "Error: RPC command \"" << rpc_command << "\" found in *both* RPC_COMMANDS_SAFE_FOR_FUZZING and RPC_COMMANDS_NOT_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n"; + std::terminate(); + } + } + const char* limit_to_rpc_command_env = std::getenv("LIMIT_TO_RPC_COMMAND"); + if (limit_to_rpc_command_env != nullptr) { + g_limit_to_rpc_command = std::string{limit_to_rpc_command_env}; + } +} + +FUZZ_TARGET_INIT(rpc, initialize_rpc) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + SetMockTime(ConsumeTime(fuzzed_data_provider)); + const std::string rpc_command = fuzzed_data_provider.ConsumeRandomLengthString(64); + if (!g_limit_to_rpc_command.empty() && rpc_command != g_limit_to_rpc_command) { + return; + } + const bool safe_for_fuzzing = std::find(RPC_COMMANDS_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_SAFE_FOR_FUZZING.end(); + if (!safe_for_fuzzing) { + return; + } + std::vector<std::string> arguments; + while (fuzzed_data_provider.ConsumeBool()) { + arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider)); + } + try { + rpc_testing_setup->CallRPC(rpc_command, arguments); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } +} diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index e87ae5b04b..b87bcf2ef5 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -43,7 +43,7 @@ FUZZ_TARGET_INIT(script, initialize_script) if (!script_opt) return; const CScript script{*script_opt}; - std::vector<unsigned char> compressed; + CompressedScript compressed; if (CompressScript(script, compressed)) { const unsigned int size = compressed[0]; compressed.erase(compressed.begin()); @@ -55,22 +55,45 @@ FUZZ_TARGET_INIT(script, initialize_script) } CTxDestination address; - (void)ExtractDestination(script, address); - TxoutType type_ret; std::vector<CTxDestination> addresses; int required_ret; - (void)ExtractDestinations(script, type_ret, addresses, required_ret); - - const FlatSigningProvider signing_provider; - (void)InferDescriptor(script, signing_provider); - - (void)IsSegWitOutput(signing_provider, script); - - (void)IsSolvable(signing_provider, script); + bool extract_destinations_ret = ExtractDestinations(script, type_ret, addresses, required_ret); + bool extract_destination_ret = ExtractDestination(script, address); + if (!extract_destinations_ret) { + assert(!extract_destination_ret); + if (type_ret == TxoutType::MULTISIG) { + assert(addresses.empty() && required_ret == 0); + } else { + assert(type_ret == TxoutType::PUBKEY || + type_ret == TxoutType::NONSTANDARD || + type_ret == TxoutType::NULL_DATA); + } + } else { + assert(required_ret >= 1 && required_ret <= 16); + assert((unsigned long)required_ret == addresses.size()); + assert(type_ret == TxoutType::MULTISIG || required_ret == 1); + } + if (type_ret == TxoutType::NONSTANDARD || type_ret == TxoutType::NULL_DATA) { + assert(!extract_destinations_ret); + } + if (!extract_destination_ret) { + assert(type_ret == TxoutType::PUBKEY || + type_ret == TxoutType::NONSTANDARD || + type_ret == TxoutType::NULL_DATA || + type_ret == TxoutType::MULTISIG); + } else { + assert(address == addresses[0]); + } + if (type_ret == TxoutType::NONSTANDARD || + type_ret == TxoutType::NULL_DATA || + type_ret == TxoutType::MULTISIG) { + assert(!extract_destination_ret); + } TxoutType which_type; bool is_standard_ret = IsStandard(script, which_type); + assert(type_ret == which_type); if (!is_standard_ret) { assert(which_type == TxoutType::NONSTANDARD || which_type == TxoutType::NULL_DATA || @@ -87,6 +110,11 @@ FUZZ_TARGET_INIT(script, initialize_script) which_type == TxoutType::NONSTANDARD); } + const FlatSigningProvider signing_provider; + (void)InferDescriptor(script, signing_provider); + (void)IsSegWitOutput(signing_provider, script); + (void)IsSolvable(signing_provider, script); + (void)RecursiveDynamicUsage(script); std::vector<std::vector<unsigned char>> solutions; @@ -115,10 +143,12 @@ FUZZ_TARGET_INIT(script, initialize_script) { const std::vector<uint8_t> bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); + CompressedScript compressed_script; + compressed_script.assign(bytes.begin(), bytes.end()); // DecompressScript(..., ..., bytes) is not guaranteed to be defined if the bytes vector is too short - if (bytes.size() >= 32) { + if (compressed_script.size() >= 32) { CScript decompressed_script; - DecompressScript(decompressed_script, fuzzed_data_provider.ConsumeIntegral<unsigned int>(), bytes); + DecompressScript(decompressed_script, fuzzed_data_provider.ConsumeIntegral<unsigned int>(), compressed_script); } } diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index cec5212f42..a80338b965 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -133,8 +133,7 @@ unsigned int ParseScriptFlags(const std::string& str) std::vector<std::string> words; boost::algorithm::split(words, str, boost::algorithm::is_any_of(",")); - for (const std::string& word : words) - { + for (const std::string& word : words) { auto it = FLAG_NAMES.find(word); if (it == FLAG_NAMES.end()) throw std::runtime_error("Unknown verification flag " + word); flags |= it->second; @@ -186,15 +185,19 @@ void Test(const std::string& str) } } -ECCVerifyHandle handle; - -} // namespace +void test_init() +{ + static ECCVerifyHandle handle; +} -FUZZ_TARGET_INIT_HIDDEN(script_assets_test_minimizer, FuzzFrameworkEmptyInitFun, /* hidden */ true) +FUZZ_TARGET_INIT_HIDDEN(script_assets_test_minimizer, test_init, /* hidden */ true) { if (buffer.size() < 2 || buffer.back() != '\n' || buffer[buffer.size() - 2] != ',') return; const std::string str((const char*)buffer.data(), buffer.size() - 2); try { Test(str); - } catch (const std::runtime_error&) {} + } catch (const std::runtime_error&) { + } } + +} // namespace diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index aa911cdeda..1278dc87d4 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -41,6 +41,10 @@ FUZZ_TARGET_INIT(script_flags, initialize_script_flags) for (unsigned i = 0; i < tx.vin.size(); ++i) { CTxOut prevout; ds >> prevout; + if (!MoneyRange(prevout.nValue)) { + // prevouts should be consensus-valid + prevout.nValue = 1; + } spent_outputs.push_back(prevout); } PrecomputedTransactionData txdata; diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index fe8d17b24a..59229987ba 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/validation.h> +#include <miner.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -77,13 +78,44 @@ void SetMempoolConstraints(ArgsManager& args, FuzzedDataProvider& fuzzed_data_pr ToString(fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, 999))); } +void Finish(FuzzedDataProvider& fuzzed_data_provider, MockedTxPool& tx_pool, CChainState& chainstate) +{ + WITH_LOCK(::cs_main, tx_pool.check(chainstate)); + { + BlockAssembler::Options options; + options.nBlockMaxWeight = fuzzed_data_provider.ConsumeIntegralInRange(0U, MAX_BLOCK_WEIGHT); + options.blockMinFeeRate = CFeeRate{ConsumeMoney(fuzzed_data_provider)}; + auto assembler = BlockAssembler{chainstate, *static_cast<CTxMemPool*>(&tx_pool), ::Params(), options}; + auto block_template = assembler.CreateNewBlock(CScript{} << OP_TRUE); + Assert(block_template->block.vtx.size() >= 1); + } + const auto info_all = tx_pool.infoAll(); + if (!info_all.empty()) { + const auto& tx_to_remove = *PickValue(fuzzed_data_provider, info_all).tx; + WITH_LOCK(tx_pool.cs, tx_pool.removeRecursive(tx_to_remove, /* dummy */ MemPoolRemovalReason::BLOCK)); + std::vector<uint256> all_txids; + tx_pool.queryHashes(all_txids); + assert(all_txids.size() < info_all.size()); + WITH_LOCK(::cs_main, tx_pool.check(chainstate)); + } + SyncWithValidationInterfaceQueue(); +} + +void MockTime(FuzzedDataProvider& fuzzed_data_provider, const CChainState& chainstate) +{ + const auto time = ConsumeTime(fuzzed_data_provider, + chainstate.m_chain.Tip()->GetMedianTimePast() + 1, + std::numeric_limits<decltype(chainstate.m_chain.Tip()->nTime)>::max()); + SetMockTime(time); +} + FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const auto& node = g_setup->m_node; auto& chainstate = node.chainman->ActiveChainstate(); - SetMockTime(ConsumeTime(fuzzed_data_provider)); + MockTime(fuzzed_data_provider, chainstate); SetMempoolConstraints(*node.args, fuzzed_data_provider); // All RBF-spendable outpoints @@ -163,7 +195,7 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) }(); if (fuzzed_data_provider.ConsumeBool()) { - SetMockTime(ConsumeTime(fuzzed_data_provider)); + MockTime(fuzzed_data_provider, chainstate); } if (fuzzed_data_provider.ConsumeBool()) { SetMempoolConstraints(*node.args, fuzzed_data_provider); @@ -237,23 +269,17 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) } } } - WITH_LOCK(::cs_main, tx_pool.check(chainstate)); - const auto info_all = tx_pool.infoAll(); - if (!info_all.empty()) { - const auto& tx_to_remove = *PickValue(fuzzed_data_provider, info_all).tx; - WITH_LOCK(tx_pool.cs, tx_pool.removeRecursive(tx_to_remove, /* dummy */ MemPoolRemovalReason::BLOCK)); - std::vector<uint256> all_txids; - tx_pool.queryHashes(all_txids); - assert(all_txids.size() < info_all.size()); - WITH_LOCK(::cs_main, tx_pool.check(chainstate)); - } - SyncWithValidationInterfaceQueue(); + Finish(fuzzed_data_provider, tx_pool, chainstate); } FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const auto& node = g_setup->m_node; + auto& chainstate = node.chainman->ActiveChainstate(); + + MockTime(fuzzed_data_provider, chainstate); + SetMempoolConstraints(*node.args, fuzzed_data_provider); std::vector<uint256> txids; for (const auto& outpoint : g_outpoints_coinbase_init_mature) { @@ -265,11 +291,29 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) txids.push_back(ConsumeUInt256(fuzzed_data_provider)); } - CTxMemPool tx_pool{/* estimator */ nullptr, /* check_ratio */ 1}; + CTxMemPool tx_pool_{/* estimator */ nullptr, /* check_ratio */ 1}; + MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); while (fuzzed_data_provider.ConsumeBool()) { const auto mut_tx = ConsumeTransaction(fuzzed_data_provider, txids); + if (fuzzed_data_provider.ConsumeBool()) { + MockTime(fuzzed_data_provider, chainstate); + } + if (fuzzed_data_provider.ConsumeBool()) { + SetMempoolConstraints(*node.args, fuzzed_data_provider); + } + if (fuzzed_data_provider.ConsumeBool()) { + tx_pool.RollingFeeUpdate(); + } + if (fuzzed_data_provider.ConsumeBool()) { + const auto& txid = fuzzed_data_provider.ConsumeBool() ? + mut_tx.GetHash() : + PickValue(fuzzed_data_provider, txids); + const auto delta = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-50 * COIN, +50 * COIN); + tx_pool.PrioritiseTransaction(txid, delta); + } + const auto tx = MakeTransactionRef(mut_tx); const bool bypass_limits = fuzzed_data_provider.ConsumeBool(); ::fRequireStandard = fuzzed_data_provider.ConsumeBool(); @@ -278,8 +322,7 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) if (accepted) { txids.push_back(tx->GetHash()); } - - SyncWithValidationInterfaceQueue(); } + Finish(fuzzed_data_provider, tx_pool, chainstate); } } // namespace diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index b8d846a995..574b694d1a 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -5,6 +5,7 @@ #include <test/fuzz/util.h> #include <test/util/script.h> #include <util/rbf.h> +#include <util/time.h> #include <version.h> FuzzedSock::FuzzedSock(FuzzedDataProvider& fuzzed_data_provider) @@ -216,6 +217,14 @@ void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, bool init_v } } +int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider, const std::optional<int64_t>& min, const std::optional<int64_t>& max) noexcept +{ + // Avoid t=0 (1970-01-01T00:00:00Z) since SetMockTime(0) disables mocktime. + static const int64_t time_min = ParseISO8601DateTime("1970-01-01T00:00:01Z"); + static const int64_t time_max = ParseISO8601DateTime("9999-12-31T23:59:59Z"); + return fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(min.value_or(time_min), max.value_or(time_max)); +} + CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, const std::optional<std::vector<uint256>>& prevout_txids, const int max_num_in, const int max_num_out) noexcept { CMutableTransaction tx_mut; @@ -267,13 +276,13 @@ CScriptWitness ConsumeScriptWitness(FuzzedDataProvider& fuzzed_data_provider, co return ret; } -CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length, const bool maybe_p2wsh) noexcept +CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length, const bool maybe_p2wsh) noexcept { const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length); CScript r_script{b.begin(), b.end()}; if (maybe_p2wsh && fuzzed_data_provider.ConsumeBool()) { uint256 script_hash; - CSHA256().Write(&r_script[0], r_script.size()).Finalize(script_hash.begin()); + CSHA256().Write(r_script.data(), r_script.size()).Finalize(script_hash.begin()); r_script.clear(); r_script << OP_0 << ToByteVector(script_hash); } diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 8f4f87fbdc..1a83f7a57a 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -26,7 +26,6 @@ #include <test/util/net.h> #include <txmempool.h> #include <uint256.h> -#include <util/time.h> #include <version.h> #include <algorithm> @@ -58,18 +57,20 @@ auto& PickValue(FuzzedDataProvider& fuzzed_data_provider, Collection& col) return *it; } -[[nodiscard]] inline std::vector<uint8_t> ConsumeRandomLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +[[nodiscard]] inline std::vector<uint8_t> ConsumeRandomLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept { - const std::string s = fuzzed_data_provider.ConsumeRandomLengthString(max_length); + const std::string s = max_length ? + fuzzed_data_provider.ConsumeRandomLengthString(*max_length) : + fuzzed_data_provider.ConsumeRandomLengthString(); return {s.begin(), s.end()}; } -[[nodiscard]] inline std::vector<bool> ConsumeRandomLengthBitVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +[[nodiscard]] inline std::vector<bool> ConsumeRandomLengthBitVector(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept { return BytesToBits(ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length)); } -[[nodiscard]] inline CDataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +[[nodiscard]] inline CDataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept { return CDataStream{ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length), SER_NETWORK, INIT_PROTO_VERSION}; } @@ -96,7 +97,7 @@ template <typename T> } template <typename T> -[[nodiscard]] inline std::optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +[[nodiscard]] inline std::optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept { const std::vector<uint8_t> buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length); CDataStream ds{buffer, SER_NETWORK, INIT_PROTO_VERSION}; @@ -127,19 +128,13 @@ template <typename WeakEnumType, size_t size> return fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, MAX_MONEY); } -[[nodiscard]] inline int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - // Avoid t=0 (1970-01-01T00:00:00Z) since SetMockTime(0) is a no-op. - static const int64_t time_min = ParseISO8601DateTime("1970-01-01T00:00:01Z"); - static const int64_t time_max = ParseISO8601DateTime("9999-12-31T23:59:59Z"); - return fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(time_min, time_max); -} +[[nodiscard]] int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider, const std::optional<int64_t>& min = std::nullopt, const std::optional<int64_t>& max = std::nullopt) noexcept; [[nodiscard]] CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, const std::optional<std::vector<uint256>>& prevout_txids, const int max_num_in = 10, const int max_num_out = 10) noexcept; [[nodiscard]] CScriptWitness ConsumeScriptWitness(FuzzedDataProvider& fuzzed_data_provider, const size_t max_stack_elem_size = 32) noexcept; -[[nodiscard]] CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096, const bool maybe_p2wsh = false) noexcept; +[[nodiscard]] CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt, const bool maybe_p2wsh = false) noexcept; [[nodiscard]] uint32_t ConsumeSequence(FuzzedDataProvider& fuzzed_data_provider) noexcept; diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 4dc0dd5f51..44fbfa5970 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -98,7 +98,7 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) // TxoutType::WITNESS_V0_SCRIPTHASH uint256 scriptHash; - CSHA256().Write(&redeemScript[0], redeemScript.size()) + CSHA256().Write(redeemScript.data(), redeemScript.size()) .Finalize(scriptHash.begin()); s.clear(); @@ -199,23 +199,20 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) s.clear(); s << ToByteVector(pubkey) << OP_CHECKSIG; BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(std::get_if<PKHash>(&address) && - *std::get_if<PKHash>(&address) == PKHash(pubkey)); + BOOST_CHECK(std::get<PKHash>(address) == PKHash(pubkey)); // TxoutType::PUBKEYHASH s.clear(); s << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(std::get_if<PKHash>(&address) && - *std::get_if<PKHash>(&address) == PKHash(pubkey)); + BOOST_CHECK(std::get<PKHash>(address) == PKHash(pubkey)); // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script s.clear(); s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(std::get_if<ScriptHash>(&address) && - *std::get_if<ScriptHash>(&address) == ScriptHash(redeemScript)); + BOOST_CHECK(std::get<ScriptHash>(address) == ScriptHash(redeemScript)); // TxoutType::MULTISIG s.clear(); @@ -233,7 +230,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) BOOST_CHECK(ExtractDestination(s, address)); WitnessV0KeyHash keyhash; CHash160().Write(pubkey).Finalize(keyhash); - BOOST_CHECK(std::get_if<WitnessV0KeyHash>(&address) && *std::get_if<WitnessV0KeyHash>(&address) == keyhash); + BOOST_CHECK(std::get<WitnessV0KeyHash>(address) == keyhash); // TxoutType::WITNESS_V0_SCRIPTHASH s.clear(); @@ -241,7 +238,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) CSHA256().Write(redeemScript.data(), redeemScript.size()).Finalize(scripthash.begin()); s << OP_0 << ToByteVector(scripthash); BOOST_CHECK(ExtractDestination(s, address)); - BOOST_CHECK(std::get_if<WitnessV0ScriptHash>(&address) && *std::get_if<WitnessV0ScriptHash>(&address) == scripthash); + BOOST_CHECK(std::get<WitnessV0ScriptHash>(address) == scripthash); // TxoutType::WITNESS_UNKNOWN with unknown version s.clear(); @@ -251,7 +248,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) unk.length = 33; unk.version = 1; std::copy(pubkey.begin(), pubkey.end(), unk.program); - BOOST_CHECK(std::get_if<WitnessUnknown>(&address) && *std::get_if<WitnessUnknown>(&address) == unk); + BOOST_CHECK(std::get<WitnessUnknown>(address) == unk); } BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) @@ -275,8 +272,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEY); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(std::get_if<PKHash>(&addresses[0]) && - *std::get_if<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); + BOOST_CHECK(std::get<PKHash>(addresses[0]) == PKHash(pubkeys[0])); // TxoutType::PUBKEYHASH s.clear(); @@ -285,8 +281,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEYHASH); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(std::get_if<PKHash>(&addresses[0]) && - *std::get_if<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); + BOOST_CHECK(std::get<PKHash>(addresses[0]) == PKHash(pubkeys[0])); // TxoutType::SCRIPTHASH CScript redeemScript(s); // initialize with leftover P2PKH script @@ -296,8 +291,7 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::SCRIPTHASH); BOOST_CHECK_EQUAL(addresses.size(), 1U); BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(std::get_if<ScriptHash>(&addresses[0]) && - *std::get_if<ScriptHash>(&addresses[0]) == ScriptHash(redeemScript)); + BOOST_CHECK(std::get<ScriptHash>(addresses[0]) == ScriptHash(redeemScript)); // TxoutType::MULTISIG s.clear(); @@ -309,10 +303,8 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) BOOST_CHECK_EQUAL(whichType, TxoutType::MULTISIG); BOOST_CHECK_EQUAL(addresses.size(), 2U); BOOST_CHECK_EQUAL(nRequired, 2); - BOOST_CHECK(std::get_if<PKHash>(&addresses[0]) && - *std::get_if<PKHash>(&addresses[0]) == PKHash(pubkeys[0])); - BOOST_CHECK(std::get_if<PKHash>(&addresses[1]) && - *std::get_if<PKHash>(&addresses[1]) == PKHash(pubkeys[1])); + BOOST_CHECK(std::get<PKHash>(addresses[0]) == PKHash(pubkeys[0])); + BOOST_CHECK(std::get<PKHash>(addresses[1]) == PKHash(pubkeys[1])); // TxoutType::NULL_DATA s.clear(); @@ -378,7 +370,7 @@ BOOST_AUTO_TEST_CASE(script_standard_GetScriptFor_) witnessScript << OP_1 << ToByteVector(pubkeys[0]) << OP_1 << OP_CHECKMULTISIG; uint256 scriptHash; - CSHA256().Write(&witnessScript[0], witnessScript.size()) + CSHA256().Write(witnessScript.data(), witnessScript.size()) .Finalize(scriptHash.begin()); expected.clear(); diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 14cfadf75d..171234f745 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -155,10 +155,10 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript if (libconsensus_flags == flags) { int expectedSuccessCode = expect ? 1 : 0; if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), txCredit.vout[0].nValue, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), txCredit.vout[0].nValue, stream.data(), stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); } else { - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), 0, (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); - BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script_with_amount(scriptPubKey.data(), scriptPubKey.size(), 0, stream.data(), stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); + BOOST_CHECK_MESSAGE(bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), stream.data(), stream.size(), 0, libconsensus_flags, nullptr) == expectedSuccessCode, message); } } #endif @@ -224,7 +224,7 @@ struct KeyData pubkey0 = key0.GetPubKey(); pubkey0H = key0.GetPubKey(); pubkey0C = key0C.GetPubKey(); - *const_cast<unsigned char*>(&pubkey0H[0]) = 0x06 | (pubkey0H[64] & 1); + *const_cast<unsigned char*>(pubkey0H.data()) = 0x06 | (pubkey0H[64] & 1); key1.Set(vchKey1, vchKey1 + 32, false); key1C.Set(vchKey1, vchKey1 + 32, true); @@ -290,7 +290,7 @@ public: } else if (wm == WitnessMode::SH) { witscript = scriptPubKey; uint256 hash; - CSHA256().Write(&witscript[0], witscript.size()).Finalize(hash.begin()); + CSHA256().Write(witscript.data(), witscript.size()).Finalize(hash.begin()); scriptPubKey = CScript() << witnessversion << ToByteVector(hash); } if (P2SH) { @@ -774,7 +774,7 @@ BOOST_AUTO_TEST_CASE(script_build) { CScript witscript = CScript() << ToByteVector(keys.pubkey0); uint256 hash; - CSHA256().Write(&witscript[0], witscript.size()).Finalize(hash.begin()); + CSHA256().Write(witscript.data(), witscript.size()).Finalize(hash.begin()); std::vector<unsigned char> hashBytes = ToByteVector(hash); hashBytes.pop_back(); tests.push_back(TestBuilder(CScript() << OP_0 << hashBytes, @@ -1520,7 +1520,7 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_returns_true) stream << spendTx; bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), nIn, libconsensus_flags, &err); + int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), stream.data(), stream.size(), nIn, libconsensus_flags, &err); BOOST_CHECK_EQUAL(result, 1); BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_OK); } @@ -1543,7 +1543,7 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_index_err) stream << spendTx; bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), nIn, libconsensus_flags, &err); + int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), stream.data(), stream.size(), nIn, libconsensus_flags, &err); BOOST_CHECK_EQUAL(result, 0); BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_TX_INDEX); } @@ -1566,7 +1566,7 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_size) stream << spendTx; bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size() * 2, nIn, libconsensus_flags, &err); + int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), stream.data(), stream.size() * 2, nIn, libconsensus_flags, &err); BOOST_CHECK_EQUAL(result, 0); BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_TX_SIZE_MISMATCH); } @@ -1589,7 +1589,7 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_serialization) stream << 0xffffffff; bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), nIn, libconsensus_flags, &err); + int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), stream.data(), stream.size(), nIn, libconsensus_flags, &err); BOOST_CHECK_EQUAL(result, 0); BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_TX_DESERIALIZE); } @@ -1612,7 +1612,7 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_amount_required_err) stream << spendTx; bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), nIn, libconsensus_flags, &err); + int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), stream.data(), stream.size(), nIn, libconsensus_flags, &err); BOOST_CHECK_EQUAL(result, 0); BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_AMOUNT_REQUIRED); } @@ -1635,7 +1635,7 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_invalid_flags) stream << spendTx; bitcoinconsensus_error err; - int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), (const unsigned char*)&stream[0], stream.size(), nIn, libconsensus_flags, &err); + int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), stream.data(), stream.size(), nIn, libconsensus_flags, &err); BOOST_CHECK_EQUAL(result, 0); BOOST_CHECK_EQUAL(err, bitcoinconsensus_ERR_INVALID_FLAGS); } @@ -1733,7 +1733,7 @@ BOOST_AUTO_TEST_CASE(script_assets_test) size_t length = file.tellg(); file.seekg(0, std::ios::beg); std::string data(length, '\0'); - file.read(&data[0], data.size()); + file.read(data.data(), data.size()); UniValue tests = read_json(data); BOOST_CHECK(tests.isArray()); BOOST_CHECK(tests.size() > 0); diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 3079c9ff29..7af2b79f37 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -112,6 +112,17 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) BOOST_CHECK_THROW(new_reader >> d, std::ios_base::failure); } +BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue) +{ + std::vector<uint8_t> data{0x82, 0xa7, 0x31}; + VectorReader reader(SER_NETWORK, INIT_PROTO_VERSION, data, /* pos= */ 0); + uint32_t varint = 0; + // Deserialize into r-value + reader >> VARINT(varint); + BOOST_CHECK_EQUAL(varint, 54321); + BOOST_CHECK(reader.empty()); +} + BOOST_AUTO_TEST_CASE(bitstream_reader_writer) { CDataStream data(SER_NETWORK, INIT_PROTO_VERSION); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 04b908829b..534d28e5de 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -1759,6 +1759,15 @@ BOOST_AUTO_TEST_CASE(test_ParseFixedPoint) BOOST_CHECK(!ParseFixedPoint("1.1e", 8, &amount)); BOOST_CHECK(!ParseFixedPoint("1.1e-", 8, &amount)); BOOST_CHECK(!ParseFixedPoint("1.", 8, &amount)); + + // Test with 3 decimal places for fee rates in sat/vB. + BOOST_CHECK(ParseFixedPoint("0.001", 3, &amount)); + BOOST_CHECK_EQUAL(amount, CAmount{1}); + BOOST_CHECK(!ParseFixedPoint("0.0009", 3, &amount)); + BOOST_CHECK(!ParseFixedPoint("31.00100001", 3, &amount)); + BOOST_CHECK(!ParseFixedPoint("31.0011", 3, &amount)); + BOOST_CHECK(!ParseFixedPoint("31.99999999", 3, &amount)); + BOOST_CHECK(!ParseFixedPoint("31.999999999999999999999", 3, &amount)); } static void TestOtherThread(fs::path dirname, std::string lockname, bool *result) diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index ab31662f97..82e70b5cdc 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -233,6 +233,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can // be found. + constexpr int snapshot_height = 110; mineBlocks(10); initial_size += 10; initial_total_coins += 10; @@ -273,6 +274,11 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) chainman.ActiveChainstate().m_from_snapshot_blockhash, *chainman.SnapshotBlockhash()); + const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params()); + const CBlockIndex* tip = chainman.ActiveTip(); + + BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx); + // To be checked against later when we try loading a subsequent snapshot. uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index bd77d74218..bacc3690a2 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -93,8 +93,7 @@ uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip) jump = DecodeJump(pos, endpos); if (jump == INVALID) break; // Jump offset straddles EOF if (bits == 0) break; // No input bits left - if (pos + jump < pos) break; // overflow - if (pos + jump >= endpos) break; // Jumping past EOF + if (int64_t{jump} >= int64_t{endpos - pos}) break; // Jumping past EOF if (ip[ip.size() - bits]) { pos += jump; } @@ -156,8 +155,7 @@ bool SanityCheckASMap(const std::vector<bool>& asmap, int bits) } else if (opcode == Instruction::JUMP) { uint32_t jump = DecodeJump(pos, endpos); if (jump == INVALID) return false; // Jump offset straddles EOF - if (pos + jump < pos) return false; // overflow - if (pos + jump > endpos) return false; // Jump out of range + if (int64_t{jump} > int64_t{endpos - pos}) return false; // Jump out of range if (bits == 0) return false; // Consuming bits past the end of the input --bits; uint32_t jump_offset = pos - begin + jump; diff --git a/src/validation.cpp b/src/validation.cpp index 2bf505e26b..52bdee699c 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -66,10 +66,6 @@ static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; /** Maximum kilobytes for transactions to store for processing during reorg */ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; -/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ -static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB -/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ -static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB /** Time to wait between writing blocks/block index to disk. */ static constexpr std::chrono::hours DATABASE_WRITE_INTERVAL{1}; /** Time to wait between flushing chainstate to disk. */ @@ -135,14 +131,9 @@ Mutex g_best_block_mutex; std::condition_variable g_best_block_cv; uint256 g_best_block; bool g_parallel_script_checks{false}; -std::atomic_bool fImporting(false); -std::atomic_bool fReindex(false); -bool fHavePruned = false; -bool fPruneMode = false; bool fRequireStandard = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; -uint64_t nPruneTarget = 0; int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; uint256 hashAssumeValid; @@ -153,22 +144,17 @@ CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); // Internal stuff namespace { CBlockIndex* pindexBestInvalid = nullptr; - - RecursiveMutex cs_LastBlockFile; - std::vector<CBlockFileInfo> vinfoBlockFile; - int nLastBlockFile = 0; - /** Global flag to indicate we should check to see if there are - * block/undo files that should be deleted. Set on startup - * or if we allocate more file space when we're in prune mode - */ - bool fCheckForPruning = false; - - /** Dirty block index entries. */ - std::set<CBlockIndex*> setDirtyBlockIndex; - - /** Dirty block file entries. */ - std::set<int> setDirtyFileInfo; -} // anon namespace +} // namespace + +// Internal stuff from blockstorage ... +extern RecursiveMutex cs_LastBlockFile; +extern std::vector<CBlockFileInfo> vinfoBlockFile; +extern int nLastBlockFile; +extern bool fCheckForPruning; +extern std::set<CBlockIndex*> setDirtyBlockIndex; +extern std::set<int> setDirtyFileInfo; +void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); +// ... TODO move fully to blockstorage CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const { @@ -205,9 +191,6 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck>* pvChecks = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); -static FlatFileSeq BlockFileSeq(); -static FlatFileSeq UndoFileSeq(); bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags) { @@ -1462,65 +1445,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, return true; } -static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) -{ - // Open history file to append - CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); - if (fileout.IsNull()) - return error("%s: OpenUndoFile failed", __func__); - - // Write index header - unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); - fileout << messageStart << nSize; - - // Write undo data - long fileOutPos = ftell(fileout.Get()); - if (fileOutPos < 0) - return error("%s: ftell failed", __func__); - pos.nPos = (unsigned int)fileOutPos; - fileout << blockundo; - - // calculate & write checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - fileout << hasher.GetHash(); - - return true; -} - -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) -{ - FlatFilePos pos = pindex->GetUndoPos(); - if (pos.IsNull()) { - return error("%s: no undo data available", __func__); - } - - // Open history file to read - CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); - if (filein.IsNull()) - return error("%s: OpenUndoFile failed", __func__); - - // Read block - uint256 hashChecksum; - CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data - try { - verifier << pindex->pprev->GetBlockHash(); - verifier >> blockundo; - filein >> hashChecksum; - } - catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); - } - - // Verify checksum - if (hashChecksum != verifier.GetHash()) - return error("%s: Checksum mismatch", __func__); - - return true; -} - -static bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str()) +bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage) { AbortNode(strMessage, userMessage); return state.Error(strMessage); @@ -1620,55 +1545,6 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -static void FlushUndoFile(int block_file, bool finalize = false) -{ - FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize); - if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { - AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); - } -} - -static void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false) -{ - LOCK(cs_LastBlockFile); - FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); - if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { - AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); - } - // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, - // e.g. during IBD or a sync after a node going offline - if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo); -} - -static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); - -static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) -{ - // Write undo information to disk - if (pindex->GetUndoPos().IsNull()) { - FlatFilePos _pos; - if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) - return error("ConnectBlock(): FindUndoPos failed"); - if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) - return AbortNode(state, "Failed to write undo data"); - // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) - // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height - // in the block file info as below; note that this does not catch the case where the undo writes are keeping up - // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in - // the FindBlockPos function - if (_pos.nFile < nLastBlockFile && static_cast<uint32_t>(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) { - FlushUndoFile(_pos.nFile, true); - } - - // update nUndoPos in block index - pindex->nUndoPos = _pos.nPos; - pindex->nStatus |= BLOCK_HAVE_UNDO; - setDirtyBlockIndex.insert(pindex); - } - - return true; -} - static CCheckQueue<CScriptCheck> scriptcheckqueue(128); void StartScriptCheckWorkerThreads(int threads_num) @@ -3102,84 +2978,6 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi } } -// TODO move to blockstorage -bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false) -{ - LOCK(cs_LastBlockFile); - - unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; - if (vinfoBlockFile.size() <= nFile) { - vinfoBlockFile.resize(nFile + 1); - } - - bool finalize_undo = false; - if (!fKnown) { - while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { - // when the undo file is keeping up with the block file, we want to flush it explicitly - // when it is lagging behind (more blocks arrive than are being connected), we let the - // undo block write case handle it - assert(std::addressof(::ChainActive()) == std::addressof(active_chain)); - finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); - nFile++; - if (vinfoBlockFile.size() <= nFile) { - vinfoBlockFile.resize(nFile + 1); - } - } - pos.nFile = nFile; - pos.nPos = vinfoBlockFile[nFile].nSize; - } - - if ((int)nFile != nLastBlockFile) { - if (!fKnown) { - LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); - } - FlushBlockFile(!fKnown, finalize_undo); - nLastBlockFile = nFile; - } - - vinfoBlockFile[nFile].AddBlock(nHeight, nTime); - if (fKnown) - vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); - else - vinfoBlockFile[nFile].nSize += nAddSize; - - if (!fKnown) { - bool out_of_space; - size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); - if (out_of_space) { - return AbortNode("Disk space is too low!", _("Disk space is too low!")); - } - if (bytes_allocated != 0 && fPruneMode) { - fCheckForPruning = true; - } - } - - setDirtyFileInfo.insert(nFile); - return true; -} - -static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) -{ - pos.nFile = nFile; - - LOCK(cs_LastBlockFile); - - pos.nPos = vinfoBlockFile[nFile].nUndoSize; - vinfoBlockFile[nFile].nUndoSize += nAddSize; - setDirtyFileInfo.insert(nFile); - - bool out_of_space; - size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); - if (out_of_space) { - return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); - } - if (bytes_allocated != 0 && fPruneMode) { - fCheckForPruning = true; - } - - return true; -} - static bool CheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true) { // Check proof of work matches claimed amount @@ -3737,18 +3535,6 @@ bool TestBlockValidity(BlockValidationState& state, * BLOCK PRUNING CODE */ -/* Calculate the amount of disk space the block & undo files currently use */ -uint64_t CalculateCurrentUsage() -{ - LOCK(cs_LastBlockFile); - - uint64_t retval = 0; - for (const CBlockFileInfo &file : vinfoBlockFile) { - retval += file.nSize + file.nUndoSize; - } - return retval; -} - void BlockManager::PruneOneBlockFile(const int fileNumber) { AssertLockHeld(cs_main); @@ -3783,17 +3569,6 @@ void BlockManager::PruneOneBlockFile(const int fileNumber) setDirtyFileInfo.insert(fileNumber); } - -void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) -{ - for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { - FlatFilePos pos(*it, 0); - fs::remove(BlockFileSeq().FileName(pos)); - fs::remove(UndoFileSeq().FileName(pos)); - LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); - } -} - void BlockManager::FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height) { assert(fPruneMode && nManualPruneHeight > 0); @@ -3888,30 +3663,6 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr nLastBlockWeCanPrune, count); } -static FlatFileSeq BlockFileSeq() -{ - return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); -} - -static FlatFileSeq UndoFileSeq() -{ - return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); -} - -FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) { - return BlockFileSeq().Open(pos, fReadOnly); -} - -/** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) { - return UndoFileSeq().Open(pos, fReadOnly); -} - -fs::path GetBlockPosFilename(const FlatFilePos &pos) -{ - return BlockFileSeq().FileName(pos); -} - CBlockIndex * BlockManager::InsertBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); @@ -4101,38 +3852,46 @@ CVerifyDB::~CVerifyDB() uiInterface.ShowProgress("", 100, false); } -bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CChainState& active_chainstate, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) +bool CVerifyDB::VerifyDB( + CChainState& chainstate, + const CChainParams& chainparams, + CCoinsView& coinsview, + int nCheckLevel, int nCheckDepth) { AssertLockHeld(cs_main); - assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); - if (active_chainstate.m_chain.Tip() == nullptr || active_chainstate.m_chain.Tip()->pprev == nullptr) + assert(std::addressof(::ChainstateActive()) == std::addressof(chainstate)); + if (chainstate.m_chain.Tip() == nullptr || chainstate.m_chain.Tip()->pprev == nullptr) return true; // Verify blocks in the best chain - if (nCheckDepth <= 0 || nCheckDepth > active_chainstate.m_chain.Height()) - nCheckDepth = active_chainstate.m_chain.Height(); + if (nCheckDepth <= 0 || nCheckDepth > chainstate.m_chain.Height()) + nCheckDepth = chainstate.m_chain.Height(); nCheckLevel = std::max(0, std::min(4, nCheckLevel)); LogPrintf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); - CCoinsViewCache coins(coinsview); + CCoinsViewCache coins(&coinsview); CBlockIndex* pindex; CBlockIndex* pindexFailure = nullptr; int nGoodTransactions = 0; BlockValidationState state; int reportDone = 0; LogPrintf("[0%%]..."); /* Continued */ - for (pindex = active_chainstate.m_chain.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { - const int percentageDone = std::max(1, std::min(99, (int)(((double)(active_chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); + + bool is_snapshot_cs = !chainstate.m_from_snapshot_blockhash.IsNull(); + + for (pindex = chainstate.m_chain.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { + const int percentageDone = std::max(1, std::min(99, (int)(((double)(chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); if (reportDone < percentageDone/10) { // report every 10% step LogPrintf("[%d%%]...", percentageDone); /* Continued */ reportDone = percentageDone/10; } uiInterface.ShowProgress(_("Verifying blocks...").translated, percentageDone, false); - if (pindex->nHeight <= active_chainstate.m_chain.Height()-nCheckDepth) + if (pindex->nHeight <= chainstate.m_chain.Height()-nCheckDepth) break; - if (fPruneMode && !(pindex->nStatus & BLOCK_HAVE_DATA)) { - // If pruning, only go back as far as we have data. + if ((fPruneMode || is_snapshot_cs) && !(pindex->nStatus & BLOCK_HAVE_DATA)) { + // If pruning or running under an assumeutxo snapshot, only go + // back as far as we have data. LogPrintf("VerifyDB(): block verification stopping at height %d (pruning, no data)\n", pindex->nHeight); break; } @@ -4154,9 +3913,11 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CChainState& active_ch } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + active_chainstate.CoinsTip().DynamicMemoryUsage()) <= active_chainstate.m_coinstip_cache_size_bytes) { + size_t curr_coins_usage = coins.DynamicMemoryUsage() + chainstate.CoinsTip().DynamicMemoryUsage(); + + if (nCheckLevel >= 3 && curr_coins_usage <= chainstate.m_coinstip_cache_size_bytes) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); - DisconnectResult res = active_chainstate.DisconnectBlock(block, pindex, coins); + DisconnectResult res = chainstate.DisconnectBlock(block, pindex, coins); if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -4170,26 +3931,26 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CChainState& active_ch if (ShutdownRequested()) return true; } if (pindexFailure) - return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", active_chainstate.m_chain.Height() - pindexFailure->nHeight + 1, nGoodTransactions); + return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", chainstate.m_chain.Height() - pindexFailure->nHeight + 1, nGoodTransactions); // store block count as we move pindex at check level >= 4 - int block_count = active_chainstate.m_chain.Height() - pindex->nHeight; + int block_count = chainstate.m_chain.Height() - pindex->nHeight; // check level 4: try reconnecting blocks if (nCheckLevel >= 4) { - while (pindex != active_chainstate.m_chain.Tip()) { - const int percentageDone = std::max(1, std::min(99, 100 - (int)(((double)(active_chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * 50))); + while (pindex != chainstate.m_chain.Tip()) { + const int percentageDone = std::max(1, std::min(99, 100 - (int)(((double)(chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * 50))); if (reportDone < percentageDone/10) { // report every 10% step LogPrintf("[%d%%]...", percentageDone); /* Continued */ reportDone = percentageDone/10; } uiInterface.ShowProgress(_("Verifying blocks...").translated, percentageDone, false); - pindex = active_chainstate.m_chain.Next(pindex); + pindex = chainstate.m_chain.Next(pindex); CBlock block; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!active_chainstate.ConnectBlock(block, state, pindex, coins, chainparams)) + if (!chainstate.ConnectBlock(block, state, pindex, coins, chainparams)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s (%s)", pindex->nHeight, pindex->GetBlockHash().ToString(), state.ToString()); if (ShutdownRequested()) return true; } @@ -4289,143 +4050,23 @@ bool CChainState::ReplayBlocks(const CChainParams& params) return true; } -//! Helper for CChainState::RewindBlockIndex -void CChainState::EraseBlockData(CBlockIndex* index) +bool CChainState::NeedsRedownload(const CChainParams& params) const { AssertLockHeld(cs_main); - assert(!m_chain.Contains(index)); // Make sure this block isn't active - - // Reduce validity - index->nStatus = std::min<unsigned int>(index->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (index->nStatus & ~BLOCK_VALID_MASK); - // Remove have-data flags. - index->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); - // Remove storage location. - index->nFile = 0; - index->nDataPos = 0; - index->nUndoPos = 0; - // Remove various other things - index->nTx = 0; - index->nChainTx = 0; - index->nSequenceId = 0; - // Make sure it gets written. - setDirtyBlockIndex.insert(index); - // Update indexes - setBlockIndexCandidates.erase(index); - auto ret = m_blockman.m_blocks_unlinked.equal_range(index->pprev); - while (ret.first != ret.second) { - if (ret.first->second == index) { - m_blockman.m_blocks_unlinked.erase(ret.first++); - } else { - ++ret.first; - } - } - // Mark parent as eligible for main chain again - if (index->pprev && index->pprev->IsValid(BLOCK_VALID_TRANSACTIONS) && index->pprev->HaveTxsDownloaded()) { - setBlockIndexCandidates.insert(index->pprev); - } -} - -bool CChainState::RewindBlockIndex(const CChainParams& params) -{ - // Note that during -reindex-chainstate we are called with an empty m_chain! - - // First erase all post-segwit blocks without witness not in the main chain, - // as this can we done without costly DisconnectTip calls. Active - // blocks will be dealt with below (releasing cs_main in between). - { - LOCK(cs_main); - for (const auto& entry : m_blockman.m_block_index) { - if (IsWitnessEnabled(entry.second->pprev, params.GetConsensus()) && !(entry.second->nStatus & BLOCK_OPT_WITNESS) && !m_chain.Contains(entry.second)) { - EraseBlockData(entry.second); - } - } - } - - // Find what height we need to reorganize to. - CBlockIndex *tip; - int nHeight = 1; - { - LOCK(cs_main); - while (nHeight <= m_chain.Height()) { - // Although SCRIPT_VERIFY_WITNESS is now generally enforced on all - // blocks in ConnectBlock, we don't need to go back and - // re-download/re-verify blocks from before segwit actually activated. - if (IsWitnessEnabled(m_chain[nHeight - 1], params.GetConsensus()) && !(m_chain[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { - break; - } - nHeight++; - } - - tip = m_chain.Tip(); - } - // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 - - BlockValidationState state; - // Loop until the tip is below nHeight, or we reach a pruned block. - while (!ShutdownRequested()) { - { - LOCK(cs_main); - LOCK(m_mempool.cs); - // Make sure nothing changed from under us (this won't happen because RewindBlockIndex runs before importing/network are active) - assert(tip == m_chain.Tip()); - if (tip == nullptr || tip->nHeight < nHeight) break; - if (fPruneMode && !(tip->nStatus & BLOCK_HAVE_DATA)) { - // If pruning, don't try rewinding past the HAVE_DATA point; - // since older blocks can't be served anyway, there's - // no need to walk further, and trying to DisconnectTip() - // will fail (and require a needless reindex/redownload - // of the blockchain). - break; - } - // Disconnect block - if (!DisconnectTip(state, params, nullptr)) { - return error("RewindBlockIndex: unable to disconnect block at height %i (%s)", tip->nHeight, state.ToString()); - } - - // Reduce validity flag and have-data flags. - // We do this after actual disconnecting, otherwise we'll end up writing the lack of data - // to disk before writing the chainstate, resulting in a failure to continue if interrupted. - // Note: If we encounter an insufficiently validated block that - // is on m_chain, it must be because we are a pruning node, and - // this block or some successor doesn't HAVE_DATA, so we were unable to - // rewind all the way. Blocks remaining on m_chain at this point - // must not have their validity reduced. - EraseBlockData(tip); - - tip = tip->pprev; - } - // Make sure the queue of validation callbacks doesn't grow unboundedly. - LimitValidationInterfaceQueue(); + // At and above params.SegwitHeight, segwit consensus rules must be validated + CBlockIndex* block{m_chain.Tip()}; + const int segwit_height{params.GetConsensus().SegwitHeight}; - // Occasionally flush state to disk. - if (!FlushStateToDisk(params, state, FlushStateMode::PERIODIC)) { - LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); - return false; - } - } - - { - LOCK(cs_main); - if (m_chain.Tip() != nullptr) { - // We can't prune block index candidates based on our tip if we have - // no tip due to m_chain being empty! - PruneBlockIndexCandidates(); - - CheckBlockIndex(params.GetConsensus()); - - // FlushStateToDisk can possibly read ::ChainActive(). Be conservative - // and skip it here, we're about to -reindex-chainstate anyway, so - // it'll get called a bunch real soon. - BlockValidationState state; - if (!FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { - LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); - return false; - } + while (block != nullptr && block->nHeight >= segwit_height) { + if (!(block->nStatus & BLOCK_OPT_WITNESS)) { + // block is insufficiently validated for a segwit client + return true; } + block = block->pprev; } - return true; + return false; } void CChainState::UnloadBlockIndex() { @@ -4854,18 +4495,6 @@ bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) return ret; } -std::string CBlockFileInfo::ToString() const -{ - return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); -} - -CBlockFileInfo* GetBlockFileInfo(size_t n) -{ - LOCK(cs_LastBlockFile); - - return &vinfoBlockFile.at(n); -} - static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) @@ -5285,14 +4914,14 @@ bool ChainstateManager::PopulateAndValidateSnapshot( return false; } - CCoinsStats stats; + CCoinsStats stats{CoinStatsHashType::HASH_SERIALIZED}; auto breakpoint_fnc = [] { /* TODO insert breakpoint here? */ }; // As above, okay to immediately release cs_main here since no other context knows // about the snapshot_chainstate. CCoinsViewDB* snapshot_coinsdb = WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsDB()); - if (!GetUTXOStats(snapshot_coinsdb, WITH_LOCK(::cs_main, return std::ref(m_blockman)), stats, CoinStatsHashType::HASH_SERIALIZED, breakpoint_fnc)) { + if (!GetUTXOStats(snapshot_coinsdb, WITH_LOCK(::cs_main, return std::ref(m_blockman)), stats, breakpoint_fnc)) { LogPrintf("[snapshot] failed to generate coins stats\n"); return false; } @@ -5346,7 +4975,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( } assert(index); - index->nChainTx = metadata.m_nchaintx; + index->nChainTx = au_data.nChainTx; snapshot_chainstate.setBlockIndexCandidates.insert(snapshot_start_block); LogPrintf("[snapshot] validated snapshot (%.2f MB)\n", diff --git a/src/validation.h b/src/validation.h index de121ab46a..8fa45c3f77 100644 --- a/src/validation.h +++ b/src/validation.h @@ -27,6 +27,7 @@ #include <serialize.h> #include <util/check.h> #include <util/hasher.h> +#include <util/translation.h> #include <atomic> #include <map> @@ -70,8 +71,6 @@ static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25; static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; -/** The maximum size of a blk?????.dat file (since 0.8) */ -static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** Maximum number of dedicated script-checking threads allowed */ static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ @@ -79,6 +78,7 @@ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; +static constexpr bool DEFAULT_COINSTATSINDEX{false}; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; /** Default for -persistmempool */ static const bool DEFAULT_PERSIST_MEMPOOL = true; @@ -112,8 +112,6 @@ typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; extern uint256 g_best_block; -extern std::atomic_bool fImporting; -extern std::atomic_bool fReindex; /** Whether there are dedicated script-checking threads running. * False indicates all script checking is done on the main threadMessageHandler thread. */ @@ -135,20 +133,9 @@ extern arith_uint256 nMinimumChainWork; /** Best header we've seen so far (used for getheaders queries' starting points). */ extern CBlockIndex *pindexBestHeader; -/** Pruning-related variables and constants */ -/** True if any block files have ever been pruned. */ -extern bool fHavePruned; -/** True if we're running in -prune mode. */ -extern bool fPruneMode; -/** Number of MiB of block files that we're trying to stay below. */ -extern uint64_t nPruneTarget; /** Documentation for argument 'checklevel'. */ extern const std::vector<std::string> CHECKLEVEL_DOC; -/** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); -/** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const FlatFilePos &pos); /** Unload database information */ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman); /** Run instances of script checking worker threads */ @@ -170,17 +157,11 @@ void StopScriptCheckWorkerThreads(); CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); +bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str{}); + /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pindex); -/** Calculate the amount of disk space the block & undo files currently use */ -uint64_t CalculateCurrentUsage(); - -/** - * Actually unlink the specified files - */ -void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); - /** Prune block files up to a given height */ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeight); @@ -195,14 +176,14 @@ struct MempoolAcceptResult { VALID, //!> Fully validated, valid. INVALID, //!> Invalid. }; - ResultType m_result_type; - TxValidationState m_state; + const ResultType m_result_type; + const TxValidationState m_state; // The following fields are only present when m_result_type = ResultType::VALID /** Mempool transactions replaced by the tx per BIP 125 rules. */ - std::optional<std::list<CTransactionRef>> m_replaced_transactions; - /** Raw base fees. */ - std::optional<CAmount> m_base_fees; + const std::optional<std::list<CTransactionRef>> m_replaced_transactions; + /** Raw base fees in satoshis. */ + const std::optional<CAmount> m_base_fees; /** Constructor for failure case */ explicit MempoolAcceptResult(TxValidationState state) @@ -212,7 +193,7 @@ struct MempoolAcceptResult { /** Constructor for success case */ explicit MempoolAcceptResult(std::list<CTransactionRef>&& replaced_txns, CAmount fees) - : m_result_type(ResultType::VALID), m_state(TxValidationState{}), + : m_result_type(ResultType::VALID), m_replaced_transactions(std::move(replaced_txns)), m_base_fees(fees) {} }; @@ -329,7 +310,12 @@ class CVerifyDB { public: CVerifyDB(); ~CVerifyDB(); - bool VerifyDB(const CChainParams& chainparams, CChainState& active_chainstate, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool VerifyDB( + CChainState& chainstate, + const CChainParams& chainparams, + CCoinsView& coinsview, + int nCheckLevel, + int nCheckDepth) EXCLUSIVE_LOCKS_REQUIRED(cs_main); }; enum DisconnectResult @@ -713,7 +699,9 @@ public: /** Replay blocks that aren't fully applied to the database. */ bool ReplayBlocks(const CChainParams& params); - bool RewindBlockIndex(const CChainParams& params) LOCKS_EXCLUDED(cs_main); + + /** Whether the chain state needs to be redownloaded due to lack of witness data */ + [[nodiscard]] bool NeedsRedownload(const CChainParams& params) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); @@ -760,9 +748,6 @@ private: bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - //! Mark a block as not having block data - void EraseBlockData(CBlockIndex* index) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void CheckForkWarningConditions() EXCLUSIVE_LOCKS_REQUIRED(cs_main); void InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -914,6 +899,8 @@ public: return m_blockman.m_block_index; } + //! @returns true if a snapshot-based chainstate is in use. Also implies + //! that a background validation chainstate is also in use. bool IsSnapshotActive() const; std::optional<uint256> SnapshotBlockhash() const; @@ -1002,9 +989,6 @@ extern VersionBitsCache versionbitscache; */ int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params); -/** Get block file info entry for one block file */ -CBlockFileInfo* GetBlockFileInfo(size_t n); - using FopenFn = std::function<FILE*(const fs::path&, const char*)>; /** Dump the mempool to disk. */ @@ -1013,12 +997,6 @@ bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function = fsbri /** Load the mempool from disk. */ bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function = fsbridge::fopen); -//! Check whether the block associated with this index entry is pruned or not. -inline bool IsBlockPruned(const CBlockIndex* pblockindex) -{ - return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); -} - /** * Return the expected assumeutxo value for a given height, if one exists. * diff --git a/src/wallet/coincontrol.cpp b/src/wallet/coincontrol.cpp index 720877ead0..598c5f082c 100644 --- a/src/wallet/coincontrol.cpp +++ b/src/wallet/coincontrol.cpp @@ -6,21 +6,7 @@ #include <util/system.h> -void CCoinControl::SetNull() +CCoinControl::CCoinControl() { - destChange = CNoDestination(); - m_change_type.reset(); - m_add_inputs = true; - fAllowOtherInputs = false; - fAllowWatchOnly = false; m_avoid_partial_spends = gArgs.GetBoolArg("-avoidpartialspends", DEFAULT_AVOIDPARTIALSPENDS); - m_avoid_address_reuse = false; - setSelected.clear(); - m_feerate.reset(); - fOverrideFeeRate = false; - m_confirm_target.reset(); - m_signal_bip125_rbf.reset(); - m_fee_mode = FeeEstimateMode::UNSET; - m_min_depth = DEFAULT_MIN_DEPTH; - m_max_depth = DEFAULT_MAX_DEPTH; } diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index d25a3fb3fa..716e1922fe 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -24,17 +24,17 @@ class CCoinControl { public: //! Custom change destination, if not set an address is generated - CTxDestination destChange; + CTxDestination destChange = CNoDestination(); //! Override the default change type if set, ignored if destChange is set std::optional<OutputType> m_change_type; //! If false, only selected inputs are used - bool m_add_inputs; + bool m_add_inputs = true; //! If false, allows unselected inputs, but requires all selected inputs be used - bool fAllowOtherInputs; + bool fAllowOtherInputs = false; //! Includes watch only addresses which are solvable - bool fAllowWatchOnly; + bool fAllowWatchOnly = false; //! Override automatic min/max checks on fee, m_feerate must be set if true - bool fOverrideFeeRate; + bool fOverrideFeeRate = false; //! Override the wallet's m_pay_tx_fee if set std::optional<CFeeRate> m_feerate; //! Override the default confirmation target if set @@ -42,22 +42,17 @@ public: //! Override the wallet's m_signal_rbf if set std::optional<bool> m_signal_bip125_rbf; //! Avoid partial use of funds sent to a given address - bool m_avoid_partial_spends; + bool m_avoid_partial_spends = DEFAULT_AVOIDPARTIALSPENDS; //! Forbids inclusion of dirty (previously used) addresses - bool m_avoid_address_reuse; + bool m_avoid_address_reuse = false; //! Fee estimation mode to control arguments to estimateSmartFee - FeeEstimateMode m_fee_mode; + FeeEstimateMode m_fee_mode = FeeEstimateMode::UNSET; //! Minimum chain depth value for coin availability int m_min_depth = DEFAULT_MIN_DEPTH; //! Maximum chain depth value for coin availability int m_max_depth = DEFAULT_MAX_DEPTH; - CCoinControl() - { - SetNull(); - } - - void SetNull(); + CCoinControl(); bool HasSelected() const { diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index f0e1addaf1..5c1b36be6e 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -15,6 +15,7 @@ static constexpr CAmount MIN_CHANGE{COIN / 100}; //! final minimum change amount after paying for fees static const CAmount MIN_FINAL_CHANGE = MIN_CHANGE/2; +/** A UTXO under consideration for use in funding a new transaction. */ class CInputCoin { public: CInputCoin(const CTransactionRef& tx, unsigned int i) @@ -56,31 +57,58 @@ public: } }; +/** Parameters for filtering which OutputGroups we may use in coin selection. + * We start by being very selective and requiring multiple confirmations and + * then get more permissive if we cannot fund the transaction. */ struct CoinEligibilityFilter { + /** Minimum number of confirmations for outputs that we sent to ourselves. + * We may use unconfirmed UTXOs sent from ourselves, e.g. change outputs. */ const int conf_mine; + /** Minimum number of confirmations for outputs received from a different + * wallet. We never spend unconfirmed foreign outputs as we cannot rely on these funds yet. */ const int conf_theirs; + /** Maximum number of unconfirmed ancestors aggregated across all UTXOs in an OutputGroup. */ const uint64_t max_ancestors; + /** Maximum number of descendants that a single UTXO in the OutputGroup may have. */ const uint64_t max_descendants; - const bool m_include_partial_groups{false}; //! Include partial destination groups when avoid_reuse and there are full groups + /** When avoid_reuse=true and there are full groups (OUTPUT_GROUP_MAX_ENTRIES), whether or not to use any partial groups.*/ + const bool m_include_partial_groups{false}; CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_ancestors) {} CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants) {} CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants, bool include_partial) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants), m_include_partial_groups(include_partial) {} }; +/** A group of UTXOs paid to the same output script. */ struct OutputGroup { + /** The list of UTXOs contained in this output group. */ std::vector<CInputCoin> m_outputs; + /** Whether the UTXOs were sent by the wallet to itself. This is relevant because we may want at + * least a certain number of confirmations on UTXOs received from outside wallets while trusting + * our own UTXOs more. */ bool m_from_me{true}; + /** The total value of the UTXOs in sum. */ CAmount m_value{0}; + /** The minimum number of confirmations the UTXOs in the group have. Unconfirmed is 0. */ int m_depth{999}; + /** The aggregated count of unconfirmed ancestors of all UTXOs in this + * group. Not deduplicated and may overestimate when ancestors are shared. */ size_t m_ancestors{0}; + /** The maximum count of descendants of a single UTXO in this output group. */ size_t m_descendants{0}; + /** The value of the UTXOs after deducting the cost of spending them at the effective feerate. */ CAmount effective_value{0}; + /** The fee to spend these UTXOs at the effective feerate. */ CAmount fee{0}; + /** The target feerate of the transaction we're trying to build. */ CFeeRate m_effective_feerate{0}; + /** The fee to spend these UTXOs at the long term feerate. */ CAmount long_term_fee{0}; + /** The feerate for spending a created change output eventually (i.e. not urgently, and thus at + * a lower feerate). Calculated using long term fee estimate. This is used to decide whether + * it could be economical to create a change output. */ CFeeRate m_long_term_feerate{0}; OutputGroup() {} diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index b50f00e7d1..251778e401 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -78,7 +78,7 @@ bool CCrypter::Encrypt(const CKeyingMaterial& vchPlaintext, std::vector<unsigned vchCiphertext.resize(vchPlaintext.size() + AES_BLOCKSIZE); AES256CBCEncrypt enc(vchKey.data(), vchIV.data(), true); - size_t nLen = enc.Encrypt(&vchPlaintext[0], vchPlaintext.size(), vchCiphertext.data()); + size_t nLen = enc.Encrypt(vchPlaintext.data(), vchPlaintext.size(), vchCiphertext.data()); if(nLen < vchPlaintext.size()) return false; vchCiphertext.resize(nLen); @@ -97,7 +97,7 @@ bool CCrypter::Decrypt(const std::vector<unsigned char>& vchCiphertext, CKeyingM vchPlaintext.resize(nLen); AES256CBCDecrypt dec(vchKey.data(), vchIV.data(), true); - nLen = dec.Decrypt(vchCiphertext.data(), vchCiphertext.size(), &vchPlaintext[0]); + nLen = dec.Decrypt(vchCiphertext.data(), vchCiphertext.size(), vchPlaintext.data()); if(nLen == 0) return false; vchPlaintext.resize(nLen); @@ -121,7 +121,7 @@ bool DecryptSecret(const CKeyingMaterial& vMasterKey, const std::vector<unsigned memcpy(chIV.data(), &nIV, WALLET_CRYPTO_IV_SIZE); if(!cKeyCrypter.SetKey(vMasterKey, chIV)) return false; - return cKeyCrypter.Decrypt(vchCiphertext, *((CKeyingMaterial*)&vchPlaintext)); + return cKeyCrypter.Decrypt(vchCiphertext, vchPlaintext); } bool DecryptKey(const CKeyingMaterial& vMasterKey, const std::vector<unsigned char>& vchCryptedSecret, const CPubKey& vchPubKey, CKey& key) diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 653dbdfc1d..a851765c57 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -469,7 +469,7 @@ RPCHelpMan importpubkey() if (!IsHex(request.params[0].get_str())) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey must be a hex string"); std::vector<unsigned char> data(ParseHex(request.params[0].get_str())); - CPubKey pubKey(data.begin(), data.end()); + CPubKey pubKey(data); if (!pubKey.IsFullyValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey is not a valid public key"); @@ -871,7 +871,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d switch (script_type) { case TxoutType::PUBKEY: { - CPubKey pubkey(solverdata[0].begin(), solverdata[0].end()); + CPubKey pubkey(solverdata[0]); import_data.used_keys.emplace(pubkey.GetID(), false); return ""; } @@ -893,7 +893,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d } case TxoutType::MULTISIG: { for (size_t i = 1; i + 1< solverdata.size(); ++i) { - CPubKey pubkey(solverdata[i].begin(), solverdata[i].end()); + CPubKey pubkey(solverdata[i]); import_data.used_keys.emplace(pubkey.GetID(), false); } return ""; @@ -997,7 +997,7 @@ static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CP throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" must be a hex string"); } auto parsed_pubkey = ParseHex(str); - CPubKey pubkey(parsed_pubkey.begin(), parsed_pubkey.end()); + CPubKey pubkey(parsed_pubkey); if (!pubkey.IsFullyValid()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" is not a valid public key"); } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 67d9d56133..7777543ece 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -216,7 +216,8 @@ static void SetFeeEstimateMode(const CWallet& wallet, CCoinControl& cc, const Un if (!estimate_mode.isNull() && estimate_mode.get_str() != "unset") { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both estimate_mode and fee_rate"); } - cc.m_feerate = CFeeRate(AmountFromValue(fee_rate), COIN); + // Fee rates in sat/vB cannot represent more than 3 significant digits. + cc.m_feerate = CFeeRate{AmountFromValue(fee_rate, /* decimals */ 3)}; if (override_min_fee) cc.fOverrideFeeRate = true; // Default RBF to true for explicit fee_rate, if unset. if (!cc.m_signal_bip125_rbf) cc.m_signal_bip125_rbf = true; @@ -540,7 +541,7 @@ static RPCHelpMan listaddressgroupings() { {RPCResult::Type::ARR, "", "", { - {RPCResult::Type::ARR, "", "", + {RPCResult::Type::ARR_FIXED, "", "", { {RPCResult::Type::STR, "address", "The bitcoin address"}, {RPCResult::Type::STR_AMOUNT, "amount", "The amount in " + CURRENCY_UNIT}, diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 6d912be019..ea045eb6d8 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -154,8 +154,8 @@ bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::v warnings.push_back(strprintf(Untranslated("WARNING: WalletBatch::Recover skipping %s: %s"), strType, strErr)); continue; } - Dbt datKey(&row.first[0], row.first.size()); - Dbt datValue(&row.second[0], row.second.size()); + Dbt datKey(row.first.data(), row.first.size()); + Dbt datValue(row.second.data(), row.second.size()); int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); if (ret2 > 0) fSuccess = false; diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 149549410c..2eb9ca5c6d 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -162,7 +162,7 @@ IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& s break; } uint160 hash; - CRIPEMD160().Write(&vSolutions[0][0], vSolutions[0].size()).Finalize(hash.begin()); + CRIPEMD160().Write(vSolutions[0].data(), vSolutions[0].size()).Finalize(hash.begin()); CScriptID scriptID = CScriptID(hash); CScript subscript; if (keystore.GetCScript(scriptID, subscript)) { diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 011d59ecaf..c8e3c8f819 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -11,6 +11,7 @@ #include <vector> #include <interfaces/chain.h> +#include <node/blockstorage.h> #include <node/context.h> #include <policy/policy.h> #include <rpc/server.h> diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 332e7b1397..3d0ef9f8c0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -603,12 +603,12 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) CKeyingMaterial _vMasterKey; _vMasterKey.resize(WALLET_CRYPTO_KEY_SIZE); - GetStrongRandBytes(&_vMasterKey[0], WALLET_CRYPTO_KEY_SIZE); + GetStrongRandBytes(_vMasterKey.data(), WALLET_CRYPTO_KEY_SIZE); CMasterKey kMasterKey; kMasterKey.vchSalt.resize(WALLET_CRYPTO_SALT_SIZE); - GetStrongRandBytes(&kMasterKey.vchSalt[0], WALLET_CRYPTO_SALT_SIZE); + GetStrongRandBytes(kMasterKey.vchSalt.data(), WALLET_CRYPTO_SALT_SIZE); CCrypter crypter; int64_t nStartTime = GetTimeMillis(); @@ -2478,7 +2478,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm } } - // remove preset inputs from vCoins + // remove preset inputs from vCoins so that Coin Selection doesn't pick them. for (std::vector<COutput>::iterator it = vCoins.begin(); it != vCoins.end() && coin_control.HasSelected();) { if (setPresetCoins.count(it->GetInputCoin())) @@ -2490,9 +2490,9 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm unsigned int limit_ancestor_count = 0; unsigned int limit_descendant_count = 0; chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); - size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); - size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); - bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); + const size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); + const size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); + const bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); // form groups from remaining coins; note that preset coins will not // automatically have their associated (same address) coins included @@ -2502,16 +2502,53 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // explicitly shuffling the outputs before processing Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); } - bool res = value_to_select <= 0 || - SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); - // because SelectCoinsMinConf clears the setCoinsRet, we now add the possible inputs to the coinset + // Coin Selection attempts to select inputs from a pool of eligible UTXOs to fund the + // transaction at a target feerate. If an attempt fails, more attempts may be made using a more + // permissive CoinEligibilityFilter. + const bool res = [&] { + // Pre-selected inputs already cover the target amount. + if (value_to_select <= 0) return true; + + // If possible, fund the transaction with confirmed UTXOs only. Prefer at least six + // confirmations on outputs received from other wallets and only spend confirmed change. + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) return true; + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) return true; + + // Fall back to using zero confirmation change (but with as few ancestors in the mempool as + // possible) if we cannot fund the transaction otherwise. We never spend unconfirmed + // outputs received from other wallets. + if (m_spend_zero_conf_change) { + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) return true; + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), + vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) { + return true; + } + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), + vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) { + return true; + } + // If partial groups are allowed, relax the requirement of spending OutputGroups (groups + // of UTXOs sent to the same address, which are obviously controlled by a single wallet) + // in their entirety. + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), + vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) { + return true; + } + // Try with unlimited ancestors/descendants. The transaction will still need to meet + // mempool ancestor/descendant policy to be accepted to mempool and broadcasted, but + // OutputGroups use heuristics that may overestimate ancestor/descendant counts. + if (!fRejectLongChains && SelectCoinsMinConf(value_to_select, + CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), true /* include_partial_groups */), + vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) { + return true; + } + } + // Coin Selection failed. + return false; + }(); + + // SelectCoinsMinConf clears setCoinsRet, so add the preset inputs from coin_control to the coinset util::insert(setCoinsRet, setPresetCoins); // add preset inputs to the total value selected diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c4acef8705..03adca7a89 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -368,7 +368,7 @@ public: CTransactionRef tx; - /* New transactions start as UNCONFIRMED. At BlockConnected, + /** New transactions start as UNCONFIRMED. At BlockConnected, * they will transition to CONFIRMED. In case of reorg, at BlockDisconnected, * they roll back to UNCONFIRMED. If we detect a conflicting transaction at * block connection, we update conflicted tx and its dependencies as CONFLICTED. @@ -383,7 +383,7 @@ public: ABANDONED }; - /* Confirmation includes tx status and a triplet of {block height/block hash/tx index in block} + /** Confirmation includes tx status and a triplet of {block height/block hash/tx index in block} * at which tx has been confirmed. All three are set to 0 if tx is unconfirmed or abandoned. * Meaning of these fields changes with CONFLICTED state where they instead point to block hash * and block height of the deepest conflicting tx. @@ -481,7 +481,7 @@ public: CAmount GetImmatureWatchOnlyCredit(const bool fUseCache = true) const; CAmount GetChange() const; - // Get the marginal bytes if spending the specified output from this transaction + /** Get the marginal bytes if spending the specified output from this transaction */ int GetSpendSize(unsigned int out, bool use_max_sig = false) const { return CalculateMaximumSignedInputSize(tx->vout[out], pwallet, use_max_sig); @@ -495,7 +495,7 @@ public: return (GetDebit(filter) > 0); } - // True if only scriptSigs are different + /** True if only scriptSigs are different */ bool IsEquivalentTo(const CWalletTx& tx) const; bool InMempool() const; @@ -503,7 +503,7 @@ public: int64_t GetTxTime() const; - // Pass this transaction to node for mempool insertion and relay to peers if flag set to true + /** Pass this transaction to node for mempool insertion and relay to peers if flag set to true */ bool SubmitMemoryPoolAndRelay(std::string& err_string, bool relay); // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct @@ -564,7 +564,15 @@ class COutput { public: const CWalletTx *tx; + + /** Index in tx->vout. */ int i; + + /** + * Depth in block chain. + * If > 0: the tx is on chain and has this many confirmations. + * If = 0: the tx is waiting confirmation. + * If < 0: a conflicting tx is on chain and has this many confirmations. */ int nDepth; /** Pre-computed estimated size of this output as a fully-signed input in a transaction. Can be -1 if it could not be calculated */ @@ -604,17 +612,30 @@ public: } }; +/** Parameters for one iteration of Coin Selection. */ struct CoinSelectionParams { + /** Toggles use of Branch and Bound instead of Knapsack solver. */ bool use_bnb = true; + /** Size of a change output in bytes, determined by the output type. */ size_t change_output_size = 0; + /** Size of the input to spend a change output in virtual bytes. */ size_t change_spend_size = 0; + /** The targeted feerate of the transaction being built. */ CFeeRate m_effective_feerate; + /** The feerate estimate used to estimate an upper bound on what should be sufficient to spend + * the change output sometime in the future. */ CFeeRate m_long_term_feerate; + /** If the cost to spend a change output at the discard feerate exceeds its value, drop it to fees. */ CFeeRate m_discard_feerate; + /** Size of the transaction before coin selection, consisting of the header and recipient + * output(s), excluding the inputs and change output(s). */ size_t tx_noinputs_size = 0; - //! Indicate that we are subtracting the fee from outputs + /** Indicate that we are subtracting the fee from outputs */ bool m_subtract_fee_outputs = false; + /** When true, always spend all (up to OUTPUT_GROUP_MAX_ENTRIES) or none of the outputs + * associated with the same address. This helps reduce privacy leaks resulting from address + * reuse. Dust outputs are not eligible to be added to output groups and thus not considered. */ bool m_avoid_partial_spends = false; CoinSelectionParams(bool use_bnb, size_t change_output_size, size_t change_spend_size, CFeeRate effective_feerate, @@ -652,7 +673,10 @@ private: //! the current wallet version: clients below this version are not able to load the wallet int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; + /** The next scheduled rebroadcast of wallet transactions. */ int64_t nNextResend = 0; + /** Whether this wallet will submit newly created transactions to the node's mempool and + * prompt rebroadcasts (see ResendWalletTransactions()). */ bool fBroadcastTransactions = false; // Local time that the tip block was received. Used to schedule wallet rebroadcasts. std::atomic<int64_t> m_best_block_time {0}; @@ -682,10 +706,10 @@ private: */ bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool fUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /* Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ + /** Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); - /* Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ + /** Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator>) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -694,6 +718,7 @@ private: * Should be called with non-zero block_hash and posInBlock if this is for a transaction that is included in a block. */ void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** WalletFlags set on this wallet. */ std::atomic<uint64_t> m_wallet_flags{0}; bool SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose); @@ -722,7 +747,7 @@ private: */ uint256 m_last_block_processed GUARDED_BY(cs_wallet); - /* Height of last block processed is used by wallet to know depth of transactions + /** Height of last block processed is used by wallet to know depth of transactions * without relying on Chain interface beyond asynchronous updates. For safety, we * initialize it to -1. Height is a pointer on node's tip and doesn't imply * that the wallet has scanned sequentially all blocks up to this one. @@ -739,7 +764,7 @@ private: bool CreateTransactionInternal(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, FeeCalculation& fee_calc_out, bool sign); public: - /* + /** * Main wallet lock. * This lock protects all the fields added by CWallet. */ @@ -753,8 +778,11 @@ public: /** * Select a set of coins such that nValueRet >= nTargetValue and at least - * all coins from coinControl are selected; Never select unconfirmed coins - * if they are not ours + * all coins from coin_control are selected; never select unconfirmed coins if they are not ours + * param@[out] setCoinsRet Populated with inputs including pre-selected inputs from + * coin_control and Coin Selection if successful. + * param@[out] nValueRet Total value of selected coins including pre-selected ones + * from coin_control and Coin Selection if successful. */ bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl& coin_control, CoinSelectionParams& coin_selection_params, bool& bnb_used) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -788,6 +816,8 @@ public: /** Interface to assert chain access */ bool HaveChain() const { return m_chain ? true : false; } + /** Map from txid to CWalletTx for all transactions this wallet is + * interested in, including received and sent transactions. */ std::map<uint256, CWalletTx> mapWallet GUARDED_BY(cs_wallet); typedef std::multimap<int64_t, CWalletTx*> TxItems; @@ -799,6 +829,10 @@ public: std::map<CTxDestination, CAddressBookData> m_address_book GUARDED_BY(cs_wallet); const CAddressBookData* FindAddressBookEntry(const CTxDestination&, bool allow_change = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** Set of Coins owned by this wallet that we won't try to spend from. A + * Coin may be locked if it has already been used to fund a transaction + * that hasn't confirmed yet. We wouldn't consider the Coin spent already, + * but also shouldn't try to use it again. */ std::set<COutPoint> setLockedCoins GUARDED_BY(cs_wallet); /** Registered interfaces::Chain::Notifications handler. */ @@ -833,6 +867,11 @@ public: * small change; This method is stochastic for some inputs and upon * completion the coin set and corresponding actual target value is * assembled + * param@[in] coins Set of UTXOs to consider. These will be categorized into + * OutputGroups and filtered using eligibility_filter before + * selecting coins. + * param@[out] setCoinsRet Populated with the coins selected if successful. + * param@[out] nValueRet Used to return the total value of selected coins. */ bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const; @@ -956,9 +995,9 @@ public: * calling CreateTransaction(); */ bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); - // Fetch the inputs and sign with SIGHASH_ALL. + /** Fetch the inputs and sign with SIGHASH_ALL. */ bool SignTransaction(CMutableTransaction& tx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - // Sign the tx given the input coins and sighash. + /** Sign the tx given the input coins and sighash. */ bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const; @@ -1015,6 +1054,8 @@ public: CFeeRate m_pay_tx_fee{DEFAULT_PAY_TX_FEE}; unsigned int m_confirm_target{DEFAULT_TX_CONFIRM_TARGET}; + /** Allow Coin Selection to pick unconfirmed UTXOs that were sent from our own wallet if it + * cannot fund the transaction otherwise. */ bool m_spend_zero_conf_change{DEFAULT_SPEND_ZEROCONF_CHANGE}; bool m_signal_rbf{DEFAULT_WALLET_RBF}; bool m_allow_fallback_fee{true}; //!< will be false if -fallbackfee=0 @@ -1025,7 +1066,12 @@ public: * Override with -fallbackfee */ CFeeRate m_fallback_fee{DEFAULT_FALLBACK_FEE}; + + /** If the cost to spend a change output at this feerate is greater than the value of the + * output itself, just drop it to fees. */ CFeeRate m_discard_rate{DEFAULT_DISCARD_FEE}; + + /** The maximum fee amount we're willing to pay to prioritize partial spend avoidance. */ CAmount m_max_aps_fee{DEFAULT_MAX_AVOIDPARTIALSPEND_FEE}; //!< note: this is absolute fee, not fee rate OutputType m_default_address_type{DEFAULT_ADDRESS_TYPE}; /** @@ -1333,10 +1379,10 @@ public: } }; -// Calculate the size of the transaction assuming all signatures are max size -// Use DummySignatureCreator, which inserts 71 byte signatures everywhere. -// NOTE: this requires that all inputs must be in mapWallet (eg the tx should -// be IsAllFromMe). +/** Calculate the size of the transaction assuming all signatures are max size +* Use DummySignatureCreator, which inserts 71 byte signatures everywhere. +* NOTE: this requires that all inputs must be in mapWallet (eg the tx should +* be IsAllFromMe). */ std::pair<int64_t, int64_t> CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, bool use_max_sig = false) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); std::pair<int64_t, int64_t> CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, bool use_max_sig = false); diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 25afa94d0f..6ae866cc07 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -168,7 +168,7 @@ bool CZMQAbstractPublishNotifier::SendZmqMessage(const char *command, const void /* send three parts, command & data & a LE 4byte sequence number */ unsigned char msgseq[sizeof(uint32_t)]; - WriteLE32(&msgseq[0], nSequence); + WriteLE32(msgseq, nSequence); int rc = zmq_send_multipart(psocket, command, strlen(command), data, size, msgseq, (size_t)sizeof(uint32_t), nullptr); if (rc == -1) return false; diff --git a/test/README.md b/test/README.md index 713fd42214..ab34ce42dc 100644 --- a/test/README.md +++ b/test/README.md @@ -273,7 +273,7 @@ Please be aware that on Linux distributions all dependencies are usually availab Individual tests can be run by directly calling the test script, e.g.: ``` -test/lint/lint-filenames.sh +test/lint/lint-files.sh ``` You can run all the shell-based lint tests by running: diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py index d13d191b20..bada791fba 100755 --- a/test/functional/feature_blockfilterindex_prune.py +++ b/test/functional/feature_blockfilterindex_prune.py @@ -33,7 +33,7 @@ class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): self.log.info("prune some blocks") pruneheight = self.nodes[0].pruneblockchain(400) - assert_equal(pruneheight, 250) + assert_equal(pruneheight, 248) self.log.info("check if we can access the tips blockfilter when we have pruned some blocks") assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index b7c2887ee8..6c51944d81 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -8,47 +8,72 @@ Test that the CHECKLOCKTIMEVERIFY soft-fork activates at (regtest) block height 1351. """ -from test_framework.blocktools import create_coinbase, create_block, create_transaction -from test_framework.messages import CTransaction, msg_block, ToHex +from test_framework.blocktools import ( + create_block, + create_coinbase, +) +from test_framework.messages import ( + CTransaction, + msg_block, +) from test_framework.p2p import P2PInterface -from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP, CScriptNum -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - hex_str_to_bytes, +from test_framework.script import ( + CScript, + CScriptNum, + OP_1NEGATE, + OP_CHECKLOCKTIMEVERIFY, + OP_DROP, ) - -from io import BytesIO +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet CLTV_HEIGHT = 1351 -def cltv_invalidate(tx): - '''Modify the signature in vin 0 of the tx to fail CLTV +# Helper function to modify a transaction by +# 1) prepending a given script to the scriptSig of vin 0 and +# 2) (optionally) modify the nSequence of vin 0 and the tx's nLockTime +def cltv_modify_tx(node, tx, prepend_scriptsig, nsequence=None, nlocktime=None): + assert_equal(len(tx.vin), 1) + if nsequence is not None: + tx.vin[0].nSequence = nsequence + tx.nLockTime = nlocktime + + tx.vin[0].scriptSig = CScript(prepend_scriptsig + list(CScript(tx.vin[0].scriptSig))) + tx.rehash() + return tx + + +def cltv_invalidate(node, tx, failure_reason): + # Modify the signature in vin 0 and nSequence/nLockTime of the tx to fail CLTV + # + # According to BIP65, OP_CHECKLOCKTIMEVERIFY can fail due the following reasons: + # 1) the stack is empty + # 2) the top item on the stack is less than 0 + # 3) the lock-time type (height vs. timestamp) of the top stack item and the + # nLockTime field are not the same + # 4) the top stack item is greater than the transaction's nLockTime field + # 5) the nSequence field of the txin is 0xffffffff + assert failure_reason in range(5) + scheme = [ + # | Script to prepend to scriptSig | nSequence | nLockTime | + # +-------------------------------------------------+------------+--------------+ + [[OP_CHECKLOCKTIMEVERIFY], None, None], + [[OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP], None, None], + [[CScriptNum(1000), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0, 1296688602], # timestamp of genesis block + [[CScriptNum(1000), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0, 500], + [[CScriptNum(500), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0xffffffff, 500], + ][failure_reason] + + return cltv_modify_tx(node, tx, prepend_scriptsig=scheme[0], nsequence=scheme[1], nlocktime=scheme[2]) - Prepends -1 CLTV DROP in the scriptSig itself. - - TODO: test more ways that transactions using CLTV could be invalid (eg - locktime requirements fail, sequence time requirements fail, etc). - ''' - tx.vin[0].scriptSig = CScript([OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP] + - list(CScript(tx.vin[0].scriptSig))) def cltv_validate(node, tx, height): - '''Modify the signature in vin 0 of the tx to pass CLTV - Prepends <height> CLTV DROP in the scriptSig, and sets - the locktime to height''' - tx.vin[0].nSequence = 0 - tx.nLockTime = height - - # Need to re-sign, since nSequence and nLockTime changed - signed_result = node.signrawtransactionwithwallet(ToHex(tx)) - new_tx = CTransaction() - new_tx.deserialize(BytesIO(hex_str_to_bytes(signed_result['hex']))) + # Modify the signature in vin 0 and nSequence/nLockTime of the tx to pass CLTV + scheme = [[CScriptNum(height), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0, height] - new_tx.vin[0].scriptSig = CScript([CScriptNum(height), OP_CHECKLOCKTIMEVERIFY, OP_DROP] + - list(CScript(new_tx.vin[0].scriptSig))) - return new_tx + return cltv_modify_tx(node, tx, prepend_scriptsig=scheme[0], nsequence=scheme[1], nlocktime=scheme[2]) class BIP65Test(BitcoinTestFramework): @@ -62,12 +87,8 @@ class BIP65Test(BitcoinTestFramework): self.setup_clean_chain = True self.rpc_timeout = 480 - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def test_cltv_info(self, *, is_active): - assert_equal(self.nodes[0].getblockchaininfo()['softforks']['bip65'], - { + assert_equal(self.nodes[0].getblockchaininfo()['softforks']['bip65'], { "active": is_active, "height": CLTV_HEIGHT, "type": "buried", @@ -76,25 +97,28 @@ class BIP65Test(BitcoinTestFramework): def run_test(self): peer = self.nodes[0].add_p2p_connection(P2PInterface()) + wallet = MiniWallet(self.nodes[0], raw_script=True) self.test_cltv_info(is_active=False) self.log.info("Mining %d blocks", CLTV_HEIGHT - 2) - self.coinbase_txids = [self.nodes[0].getblock(b)['tx'][0] for b in self.nodes[0].generate(CLTV_HEIGHT - 2)] - self.nodeaddress = self.nodes[0].getnewaddress() + wallet.generate(10) + self.nodes[0].generate(CLTV_HEIGHT - 2 - 10) - self.log.info("Test that an invalid-according-to-CLTV transaction can still appear in a block") + self.log.info("Test that invalid-according-to-CLTV transactions can still appear in a block") - spendtx = create_transaction(self.nodes[0], self.coinbase_txids[0], - self.nodeaddress, amount=1.0) - cltv_invalidate(spendtx) - spendtx.rehash() + # create one invalid tx per CLTV failure reason (5 in total) and collect them + invalid_ctlv_txs = [] + for i in range(5): + spendtx = wallet.create_self_transfer(from_node=self.nodes[0])['tx'] + spendtx = cltv_invalidate(self.nodes[0], spendtx, i) + invalid_ctlv_txs.append(spendtx) tip = self.nodes[0].getbestblockhash() block_time = self.nodes[0].getblockheader(tip)['mediantime'] + 1 block = create_block(int(tip, 16), create_coinbase(CLTV_HEIGHT - 1), block_time) block.nVersion = 3 - block.vtx.append(spendtx) + block.vtx.extend(invalid_ctlv_txs) block.hashMerkleRoot = block.calc_merkle_root() block.solve() @@ -115,39 +139,47 @@ class BIP65Test(BitcoinTestFramework): assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) peer.sync_with_ping() - self.log.info("Test that invalid-according-to-cltv transactions cannot appear in a block") + self.log.info("Test that invalid-according-to-CLTV transactions cannot appear in a block") block.nVersion = 4 - - spendtx = create_transaction(self.nodes[0], self.coinbase_txids[1], - self.nodeaddress, amount=1.0) - cltv_invalidate(spendtx) - spendtx.rehash() - - # First we show that this tx is valid except for CLTV by getting it - # rejected from the mempool for exactly that reason. - assert_equal( - [{ - 'txid': spendtx.hash, - 'wtxid': spendtx.getwtxid(), - 'allowed': False, - 'reject-reason': 'non-mandatory-script-verify-flag (Negative locktime)', - }], - self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0), - ) - - # Now we verify that a block with this transaction is also invalid. - block.vtx.append(spendtx) - block.hashMerkleRoot = block.calc_merkle_root() - block.solve() - - with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputScripts on {} failed with non-mandatory-script-verify-flag (Negative locktime)'.format(block.vtx[-1].hash)]): - peer.send_and_ping(msg_block(block)) - assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - peer.sync_with_ping() + block.vtx.append(CTransaction()) # dummy tx after coinbase that will be replaced later + + # create and test one invalid tx per CLTV failure reason (5 in total) + for i in range(5): + spendtx = wallet.create_self_transfer(from_node=self.nodes[0])['tx'] + spendtx = cltv_invalidate(self.nodes[0], spendtx, i) + + expected_cltv_reject_reason = [ + "non-mandatory-script-verify-flag (Operation not valid with the current stack size)", + "non-mandatory-script-verify-flag (Negative locktime)", + "non-mandatory-script-verify-flag (Locktime requirement not satisfied)", + "non-mandatory-script-verify-flag (Locktime requirement not satisfied)", + "non-mandatory-script-verify-flag (Locktime requirement not satisfied)", + ][i] + # First we show that this tx is valid except for CLTV by getting it + # rejected from the mempool for exactly that reason. + assert_equal( + [{ + 'txid': spendtx.hash, + 'wtxid': spendtx.getwtxid(), + 'allowed': False, + 'reject-reason': expected_cltv_reject_reason, + }], + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0), + ) + + # Now we verify that a block with this transaction is also invalid. + block.vtx[1] = spendtx + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + + with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputScripts on {} failed with {}'.format( + block.vtx[-1].hash, expected_cltv_reject_reason)]): + peer.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) + peer.sync_with_ping() self.log.info("Test that a version 4 block with a valid-according-to-CLTV transaction is accepted") spendtx = cltv_validate(self.nodes[0], spendtx, CLTV_HEIGHT - 1) - spendtx.rehash() block.vtx.pop(1) block.vtx.append(spendtx) diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py new file mode 100755 index 0000000000..d3adde5cc5 --- /dev/null +++ b/test/functional/feature_coinstatsindex.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test coinstatsindex across nodes. + +Test that the values returned by gettxoutsetinfo are consistent +between a node running the coinstatsindex and a node without +the index. +""" + +from decimal import Decimal + +from test_framework.blocktools import ( + create_block, + create_coinbase, +) +from test_framework.messages import ( + COIN, + COutPoint, + CTransaction, + CTxIn, + CTxOut, + ToHex, +) +from test_framework.script import ( + CScript, + OP_FALSE, + OP_RETURN, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + try_rpc, +) + +class CoinStatsIndexTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + self.supports_cli = False + self.extra_args = [ + [], + ["-coinstatsindex"] + ] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self._test_coin_stats_index() + self._test_use_index_option() + self._test_reorg_index() + self._test_index_rejects_hash_serialized() + + def block_sanity_check(self, block_info): + block_subsidy = 50 + assert_equal( + block_info['prevout_spent'] + block_subsidy, + block_info['new_outputs_ex_coinbase'] + block_info['coinbase'] + block_info['unspendable'] + ) + + def _test_coin_stats_index(self): + node = self.nodes[0] + index_node = self.nodes[1] + # Both none and muhash options allow the usage of the index + index_hash_options = ['none', 'muhash'] + + # Generate a normal transaction and mine it + node.generate(101) + address = self.nodes[0].get_deterministic_priv_key().address + node.sendtoaddress(address=address, amount=10, subtractfeefromamount=True) + node.generate(1) + + self.sync_blocks(timeout=120) + + self.log.info("Test that gettxoutsetinfo() output is consistent with or without coinstatsindex option") + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", node.gettxoutsetinfo)) + res0 = node.gettxoutsetinfo('none') + + # The fields 'disk_size' and 'transactions' do not exist on the index + del res0['disk_size'], res0['transactions'] + + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + for hash_option in index_hash_options: + res1 = index_node.gettxoutsetinfo(hash_option) + # The fields 'block_info' and 'total_unspendable_amount' only exist on the index + del res1['block_info'], res1['total_unspendable_amount'] + res1.pop('muhash', None) + + # Everything left should be the same + assert_equal(res1, res0) + + self.log.info("Test that gettxoutsetinfo() can get fetch data on specific heights with index") + + # Generate a new tip + node.generate(5) + + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + for hash_option in index_hash_options: + # Fetch old stats by height + res2 = index_node.gettxoutsetinfo(hash_option, 102) + del res2['block_info'], res2['total_unspendable_amount'] + res2.pop('muhash', None) + assert_equal(res0, res2) + + # Fetch old stats by hash + res3 = index_node.gettxoutsetinfo(hash_option, res0['bestblock']) + del res3['block_info'], res3['total_unspendable_amount'] + res3.pop('muhash', None) + assert_equal(res0, res3) + + # It does not work without coinstatsindex + assert_raises_rpc_error(-8, "Querying specific block heights requires coinstatsindex", node.gettxoutsetinfo, hash_option, 102) + + self.log.info("Test gettxoutsetinfo() with index and verbose flag") + + for hash_option in index_hash_options: + # Genesis block is unspendable + res4 = index_node.gettxoutsetinfo(hash_option, 0) + assert_equal(res4['total_unspendable_amount'], 50) + assert_equal(res4['block_info'], { + 'unspendable': 50, + 'prevout_spent': 0, + 'new_outputs_ex_coinbase': 0, + 'coinbase': 0, + 'unspendables': { + 'genesis_block': 50, + 'bip30': 0, + 'scripts': 0, + 'unclaimed_rewards': 0 + } + }) + self.block_sanity_check(res4['block_info']) + + # Test an older block height that included a normal tx + res5 = index_node.gettxoutsetinfo(hash_option, 102) + assert_equal(res5['total_unspendable_amount'], 50) + assert_equal(res5['block_info'], { + 'unspendable': 0, + 'prevout_spent': 50, + 'new_outputs_ex_coinbase': Decimal('49.99995560'), + 'coinbase': Decimal('50.00004440'), + 'unspendables': { + 'genesis_block': 0, + 'bip30': 0, + 'scripts': 0, + 'unclaimed_rewards': 0 + } + }) + self.block_sanity_check(res5['block_info']) + + # Generate and send a normal tx with two outputs + tx1_inputs = [] + tx1_outputs = {self.nodes[0].getnewaddress(): 21, self.nodes[0].getnewaddress(): 42} + raw_tx1 = self.nodes[0].createrawtransaction(tx1_inputs, tx1_outputs) + funded_tx1 = self.nodes[0].fundrawtransaction(raw_tx1) + signed_tx1 = self.nodes[0].signrawtransactionwithwallet(funded_tx1['hex']) + tx1_txid = self.nodes[0].sendrawtransaction(signed_tx1['hex']) + + # Find the right position of the 21 BTC output + tx1_final = self.nodes[0].gettransaction(tx1_txid) + for output in tx1_final['details']: + if output['amount'] == Decimal('21.00000000') and output['category'] == 'receive': + n = output['vout'] + + # Generate and send another tx with an OP_RETURN output (which is unspendable) + tx2 = CTransaction() + tx2.vin.append(CTxIn(COutPoint(int(tx1_txid, 16), n), b'')) + tx2.vout.append(CTxOut(int(20.99 * COIN), CScript([OP_RETURN] + [OP_FALSE]*30))) + tx2_hex = self.nodes[0].signrawtransactionwithwallet(ToHex(tx2))['hex'] + self.nodes[0].sendrawtransaction(tx2_hex) + + # Include both txs in a block + self.nodes[0].generate(1) + self.sync_all() + + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + for hash_option in index_hash_options: + # Check all amounts were registered correctly + res6 = index_node.gettxoutsetinfo(hash_option, 108) + assert_equal(res6['total_unspendable_amount'], Decimal('70.98999999')) + assert_equal(res6['block_info'], { + 'unspendable': Decimal('20.98999999'), + 'prevout_spent': 111, + 'new_outputs_ex_coinbase': Decimal('89.99993620'), + 'coinbase': Decimal('50.01006381'), + 'unspendables': { + 'genesis_block': 0, + 'bip30': 0, + 'scripts': Decimal('20.98999999'), + 'unclaimed_rewards': 0 + } + }) + self.block_sanity_check(res6['block_info']) + + # Create a coinbase that does not claim full subsidy and also + # has two outputs + cb = create_coinbase(109, nValue=35) + cb.vout.append(CTxOut(5 * COIN, CScript([OP_FALSE]))) + cb.rehash() + + # Generate a block that includes previous coinbase + tip = self.nodes[0].getbestblockhash() + block_time = self.nodes[0].getblock(tip)['time'] + 1 + block = create_block(int(tip, 16), cb, block_time) + block.solve() + self.nodes[0].submitblock(ToHex(block)) + self.sync_all() + + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + for hash_option in index_hash_options: + res7 = index_node.gettxoutsetinfo(hash_option, 109) + assert_equal(res7['total_unspendable_amount'], Decimal('80.98999999')) + assert_equal(res7['block_info'], { + 'unspendable': 10, + 'prevout_spent': 0, + 'new_outputs_ex_coinbase': 0, + 'coinbase': 40, + 'unspendables': { + 'genesis_block': 0, + 'bip30': 0, + 'scripts': 0, + 'unclaimed_rewards': 10 + } + }) + self.block_sanity_check(res7['block_info']) + + self.log.info("Test that the index is robust across restarts") + + res8 = index_node.gettxoutsetinfo('muhash') + self.restart_node(1, extra_args=self.extra_args[1]) + res9 = index_node.gettxoutsetinfo('muhash') + assert_equal(res8, res9) + + index_node.generate(1) + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + res10 = index_node.gettxoutsetinfo('muhash') + assert(res8['txouts'] < res10['txouts']) + + def _test_use_index_option(self): + self.log.info("Test use_index option for nodes running the index") + + self.connect_nodes(0, 1) + self.nodes[0].waitforblockheight(110) + res = self.nodes[0].gettxoutsetinfo('muhash') + option_res = self.nodes[1].gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False) + del res['disk_size'], option_res['disk_size'] + assert_equal(res, option_res) + + def _test_reorg_index(self): + self.log.info("Test that index can handle reorgs") + + # Generate two block, let the index catch up, then invalidate the blocks + index_node = self.nodes[1] + reorg_blocks = index_node.generatetoaddress(2, index_node.getnewaddress()) + reorg_block = reorg_blocks[1] + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + res_invalid = index_node.gettxoutsetinfo('muhash') + index_node.invalidateblock(reorg_blocks[0]) + assert_equal(index_node.gettxoutsetinfo('muhash')['height'], 110) + + # Add two new blocks + block = index_node.generate(2)[1] + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + res = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False) + + # Test that the result of the reorged block is not returned for its old block height + res2 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=112) + assert_equal(res["bestblock"], block) + assert_equal(res["muhash"], res2["muhash"]) + assert(res["muhash"] != res_invalid["muhash"]) + + # Test that requesting reorged out block by hash is still returning correct results + res_invalid2 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=reorg_block) + assert_equal(res_invalid2["muhash"], res_invalid["muhash"]) + assert(res["muhash"] != res_invalid2["muhash"]) + + # Add another block, so we don't depend on reconsiderblock remembering which + # blocks were touched by invalidateblock + index_node.generate(1) + + # Ensure that removing and re-adding blocks yields consistent results + block = index_node.getblockhash(99) + index_node.invalidateblock(block) + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + index_node.reconsiderblock(block) + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + res3 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=112) + assert_equal(res2, res3) + + self.log.info("Test that a node aware of stale blocks syncs them as well") + node = self.nodes[0] + # Ensure the node is aware of a stale block prior to restart + node.getblock(reorg_block) + + self.restart_node(0, ["-coinstatsindex"]) + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", node.gettxoutsetinfo, 'muhash')) + assert_raises_rpc_error(-32603, "Unable to read UTXO set", node.gettxoutsetinfo, 'muhash', reorg_block) + + def _test_index_rejects_hash_serialized(self): + self.log.info("Test that the rpc raises if the legacy hash is passed with the index") + + msg = "hash_serialized_2 hash type cannot be queried for a specific block" + assert_raises_rpc_error(-8, msg, self.nodes[1].gettxoutsetinfo, hash_type='hash_serialized_2', hash_or_height=111) + + for use_index in {True, False, None}: + assert_raises_rpc_error(-8, msg, self.nodes[1].gettxoutsetinfo, hash_type='hash_serialized_2', hash_or_height=111, use_index=use_index) + + +if __name__ == '__main__': + CoinStatsIndexTest().main() diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index a0bcd9f12a..de9d0d2e80 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -165,6 +165,7 @@ class ConfArgsTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log(expected_msgs=[ "Loaded 0 addresses from peers.dat", "0 addresses found from DNS seeds", + "opencon thread start", # Ensure ThreadOpenConnections::start time is properly set ]): self.start_node(0, extra_args=['-dnsseed=1', '-fixedseeds=1', f'-mocktime={start}']) with self.nodes[0].assert_debug_log(expected_msgs=[ @@ -206,6 +207,7 @@ class ConfArgsTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log(expected_msgs=[ "Loaded 0 addresses from peers.dat", "DNS seeding disabled", + "opencon thread start", # Ensure ThreadOpenConnections::start time is properly set ]): self.start_node(0, extra_args=['-dnsseed=0', '-fixedseeds=1', '-addnode=fakenodeaddr', f'-mocktime={start}']) with self.nodes[0].assert_debug_log(expected_msgs=[ diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 4e2de1daf4..6fc8773ee3 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -166,6 +166,8 @@ class NotificationsTest(BitcoinTestFramework): # Should now verify contents of each file for tx_id, blockheight, blockhash in tx_details: fname = os.path.join(self.walletnotify_dir, notify_outputname(self.wallet, tx_id)) + # Wait for the cached writes to hit storage + self.wait_until(lambda: os.path.getsize(fname) > 0, timeout=10) with open(fname, 'rt', encoding='utf-8') as f: text = f.read() # Universal newline ensures '\n' on 'nt' diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 2cf0ef2251..b5ce18a48b 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -151,7 +151,7 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['balance'], amounts[1]) self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balances") - cli_get_info = self.nodes[0].cli('-getinfo', rpcwallet3).send_cli() + cli_get_info_keys = self.nodes[0].cli('-getinfo', rpcwallet3).send_cli().keys() assert 'balance' not in cli_get_info_keys assert 'balances' not in cli_get_info_keys diff --git a/test/functional/mempool_spend_coinbase.py b/test/functional/mempool_spend_coinbase.py index a249a73315..b900aa0b9c 100755 --- a/test/functional/mempool_spend_coinbase.py +++ b/test/functional/mempool_spend_coinbase.py @@ -20,40 +20,41 @@ from test_framework.wallet import MiniWallet class MempoolSpendCoinbaseTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.setup_clean_chain = True def run_test(self): wallet = MiniWallet(self.nodes[0]) - wallet.generate(200) - chain_height = self.nodes[0].getblockcount() - assert_equal(chain_height, 200) + # Invalidate two blocks, so that miniwallet has access to a coin that will mature in the next block + chain_height = 198 + self.nodes[0].invalidateblock(self.nodes[0].getblockhash(chain_height + 1)) + assert_equal(chain_height, self.nodes[0].getblockcount()) # Coinbase at height chain_height-100+1 ok in mempool, should # get mined. Coinbase at height chain_height-100+2 is # too immature to spend. - b = [self.nodes[0].getblockhash(n) for n in range(101, 103)] - coinbase_txids = [self.nodes[0].getblock(h)['tx'][0] for h in b] - utxo_101 = wallet.get_utxo(txid=coinbase_txids[0]) - utxo_102 = wallet.get_utxo(txid=coinbase_txids[1]) + wallet.scan_blocks(start=chain_height - 100 + 1, num=1) + utxo_mature = wallet.get_utxo() + wallet.scan_blocks(start=chain_height - 100 + 2, num=1) + utxo_immature = wallet.get_utxo() - spend_101_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_101)["txid"] + spend_mature_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_mature)["txid"] - # coinbase at height 102 should be too immature to spend + # other coinbase should be too immature to spend + immature_tx = wallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_immature, mempool_valid=False) assert_raises_rpc_error(-26, "bad-txns-premature-spend-of-coinbase", - lambda: wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_102)) + lambda: self.nodes[0].sendrawtransaction(immature_tx['hex'])) - # mempool should have just spend_101: - assert_equal(self.nodes[0].getrawmempool(), [spend_101_id]) + # mempool should have just the mature one + assert_equal(self.nodes[0].getrawmempool(), [spend_mature_id]) - # mine a block, spend_101 should get confirmed + # mine a block, mature one should get confirmed self.nodes[0].generate(1) assert_equal(set(self.nodes[0].getrawmempool()), set()) - # ... and now height 102 can be spent: - spend_102_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_102)["txid"] - assert_equal(self.nodes[0].getrawmempool(), [spend_102_id]) + # ... and now previously immature can be spent: + spend_new_id = self.nodes[0].sendrawtransaction(immature_tx['hex']) + assert_equal(self.nodes[0].getrawmempool(), [spend_new_id]) if __name__ == '__main__': diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 69821763bd..87297989ba 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -11,6 +11,7 @@ from test_framework.messages import ( NODE_NETWORK, NODE_WITNESS, msg_addr, + msg_getaddr ) from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework @@ -19,18 +20,6 @@ from test_framework.util import ( ) import time -# Keep this with length <= 10. Addresses from larger messages are not relayed. -ADDRS = [] -num_ipv4_addrs = 10 - -for i in range(num_ipv4_addrs): - addr = CAddress() - addr.time = int(time.time()) + i - addr.nServices = NODE_NETWORK | NODE_WITNESS - addr.ip = "123.123.123.{}".format(i % 256) - addr.port = 8333 + i - ADDRS.append(addr) - class AddrReceiver(P2PInterface): num_ipv4_received = 0 @@ -44,36 +33,87 @@ class AddrReceiver(P2PInterface): self.num_ipv4_received += 1 +class GetAddrStore(P2PInterface): + getaddr_received = False + num_ipv4_received = 0 + + def on_getaddr(self, message): + self.getaddr_received = True + + def on_addr(self, message): + for addr in message.addrs: + self.num_ipv4_received += 1 + + def addr_received(self): + return self.num_ipv4_received != 0 + + class AddrTest(BitcoinTestFramework): + counter = 0 + mocktime = int(time.time()) + def set_test_params(self): self.num_nodes = 1 def run_test(self): - self.log.info('Create connection that sends addr messages') - addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + self.oversized_addr_test() + self.relay_tests() + self.getaddr_tests() + self.blocksonly_mode_tests() + + def setup_addr_msg(self, num): + addrs = [] + for i in range(num): + addr = CAddress() + addr.time = self.mocktime + i + addr.nServices = NODE_NETWORK | NODE_WITNESS + addr.ip = f"123.123.123.{self.counter % 256}" + addr.port = 8333 + i + addrs.append(addr) + self.counter += 1 + msg = msg_addr() + msg.addrs = addrs + return msg + + def send_addr_msg(self, source, msg, receivers): + source.send_and_ping(msg) + # pop m_next_addr_send timer + self.mocktime += 5 * 60 + self.nodes[0].setmocktime(self.mocktime) + for peer in receivers: + peer.sync_send_with_ping() - self.log.info('Send too-large addr message') - msg.addrs = ADDRS * 101 # more than 1000 addresses in one message + def oversized_addr_test(self): + self.log.info('Send an addr message that is too large') + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + + msg = self.setup_addr_msg(1010) with self.nodes[0].assert_debug_log(['addr message size = 1010']): addr_source.send_and_ping(msg) + self.nodes[0].disconnect_p2ps() + + def relay_tests(self): + self.log.info('Test address relay') self.log.info('Check that addr message content is relayed and added to addrman') + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) num_receivers = 7 receivers = [] for _ in range(num_receivers): receivers.append(self.nodes[0].add_p2p_connection(AddrReceiver())) - msg.addrs = ADDRS + + # Keep this with length <= 10. Addresses from larger messages are not + # relayed. + num_ipv4_addrs = 10 + msg = self.setup_addr_msg(num_ipv4_addrs) with self.nodes[0].assert_debug_log( [ 'Added {} addresses from 127.0.0.1: 0 tried'.format(num_ipv4_addrs), - 'received: addr (301 bytes) peer=0', + 'received: addr (301 bytes) peer=1', ] ): - addr_source.send_and_ping(msg) - self.nodes[0].setmocktime(int(time.time()) + 30 * 60) - for receiver in receivers: - receiver.sync_with_ping() + self.send_addr_msg(addr_source, msg, receivers) total_ipv4_received = sum(r.num_ipv4_received for r in receivers) @@ -82,6 +122,92 @@ class AddrTest(BitcoinTestFramework): ipv4_branching_factor = 2 assert_equal(total_ipv4_received, num_ipv4_addrs * ipv4_branching_factor) + self.nodes[0].disconnect_p2ps() + + self.log.info('Check relay of addresses received from outbound peers') + inbound_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + full_outbound_peer = self.nodes[0].add_outbound_p2p_connection(GetAddrStore(), p2p_idx=0, connection_type="outbound-full-relay") + msg = self.setup_addr_msg(2) + self.send_addr_msg(full_outbound_peer, msg, [inbound_peer]) + self.log.info('Check that the first addr message received from an outbound peer is not relayed') + # Currently, there is a flag that prevents the first addr message received + # from a new outbound peer to be relayed to others. Originally meant to prevent + # large GETADDR responses from being relayed, it now typically affects the self-announcement + # of the outbound peer which is often sent before the GETADDR response. + assert_equal(inbound_peer.num_ipv4_received, 0) + + self.log.info('Check that subsequent addr messages sent from an outbound peer are relayed') + msg2 = self.setup_addr_msg(2) + self.send_addr_msg(full_outbound_peer, msg2, [inbound_peer]) + assert_equal(inbound_peer.num_ipv4_received, 2) + + self.log.info('Check address relay to outbound peers') + block_relay_peer = self.nodes[0].add_outbound_p2p_connection(GetAddrStore(), p2p_idx=1, connection_type="block-relay-only") + msg3 = self.setup_addr_msg(2) + self.send_addr_msg(inbound_peer, msg3, [full_outbound_peer, block_relay_peer]) + + self.log.info('Check that addresses are relayed to full outbound peers') + assert_equal(full_outbound_peer.num_ipv4_received, 2) + self.log.info('Check that addresses are not relayed to block-relay-only outbound peers') + assert_equal(block_relay_peer.num_ipv4_received, 0) + + self.nodes[0].disconnect_p2ps() + + def getaddr_tests(self): + self.log.info('Test getaddr behavior') + self.log.info('Check that we send a getaddr message upon connecting to an outbound-full-relay peer') + full_outbound_peer = self.nodes[0].add_outbound_p2p_connection(GetAddrStore(), p2p_idx=0, connection_type="outbound-full-relay") + full_outbound_peer.sync_with_ping() + assert full_outbound_peer.getaddr_received + + self.log.info('Check that we do not send a getaddr message upon connecting to a block-relay-only peer') + block_relay_peer = self.nodes[0].add_outbound_p2p_connection(GetAddrStore(), p2p_idx=1, connection_type="block-relay-only") + block_relay_peer.sync_with_ping() + assert_equal(block_relay_peer.getaddr_received, False) + + self.log.info('Check that we answer getaddr messages only from inbound peers') + inbound_peer = self.nodes[0].add_p2p_connection(GetAddrStore()) + inbound_peer.sync_with_ping() + + # Add some addresses to addrman + for i in range(1000): + first_octet = i >> 8 + second_octet = i % 256 + a = f"{first_octet}.{second_octet}.1.1" + self.nodes[0].addpeeraddress(a, 8333) + + full_outbound_peer.send_and_ping(msg_getaddr()) + block_relay_peer.send_and_ping(msg_getaddr()) + inbound_peer.send_and_ping(msg_getaddr()) + + self.mocktime += 5 * 60 + self.nodes[0].setmocktime(self.mocktime) + inbound_peer.wait_until(inbound_peer.addr_received) + + assert_equal(full_outbound_peer.num_ipv4_received, 0) + assert_equal(block_relay_peer.num_ipv4_received, 0) + assert inbound_peer.num_ipv4_received > 100 + + self.nodes[0].disconnect_p2ps() + + def blocksonly_mode_tests(self): + self.log.info('Test addr relay in -blocksonly mode') + self.restart_node(0, ["-blocksonly"]) + self.mocktime = int(time.time()) + + self.log.info('Check that we send getaddr messages') + full_outbound_peer = self.nodes[0].add_outbound_p2p_connection(GetAddrStore(), p2p_idx=0, connection_type="outbound-full-relay") + full_outbound_peer.sync_with_ping() + assert full_outbound_peer.getaddr_received + + self.log.info('Check that we relay address messages') + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + msg = self.setup_addr_msg(2) + self.send_addr_msg(addr_source, msg, [full_outbound_peer]) + assert_equal(full_outbound_peer.num_ipv4_received, 2) + + self.nodes[0].disconnect_p2ps() + if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 6584efae79..ab2556cd72 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -6,22 +6,25 @@ import time -from test_framework.blocktools import create_transaction from test_framework.messages import msg_tx from test_framework.p2p import P2PInterface, P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +from test_framework.wallet import MiniWallet class P2PBlocksOnly(BitcoinTestFramework): def set_test_params(self): + self.setup_clean_chain = True self.num_nodes = 1 self.extra_args = [["-blocksonly"]] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): + self.miniwallet = MiniWallet(self.nodes[0]) + # Add enough mature utxos to the wallet, so that all txs spend confirmed coins + self.miniwallet.generate(2) + self.nodes[0].generate(100) + self.blocksonly_mode_tests() self.blocks_relay_conn_tests() @@ -30,14 +33,14 @@ class P2PBlocksOnly(BitcoinTestFramework): assert_equal(self.nodes[0].getnetworkinfo()['localrelay'], False) self.nodes[0].add_p2p_connection(P2PInterface()) - tx, txid, tx_hex = self.check_p2p_tx_violation() + tx, txid, wtxid, tx_hex = self.check_p2p_tx_violation() self.log.info('Check that txs from rpc are not rejected and relayed to other peers') tx_relay_peer = self.nodes[0].add_p2p_connection(P2PInterface()) assert_equal(self.nodes[0].getpeerinfo()[0]['relaytxes'], True) assert_equal(self.nodes[0].testmempoolaccept([tx_hex])[0]['allowed'], True) - with self.nodes[0].assert_debug_log(['received getdata for: wtx {} peer=1'.format(txid)]): + with self.nodes[0].assert_debug_log(['received getdata for: wtx {} peer=1'.format(wtxid)]): self.nodes[0].sendrawtransaction(tx_hex) tx_relay_peer.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) @@ -79,7 +82,7 @@ class P2PBlocksOnly(BitcoinTestFramework): # Ensure we disconnect if a block-relay-only connection sends us a transaction self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=0, connection_type="block-relay-only") assert_equal(self.nodes[0].getpeerinfo()[0]['relaytxes'], False) - _, txid, tx_hex = self.check_p2p_tx_violation(index=2) + _, txid, _, tx_hex = self.check_p2p_tx_violation(index=2) self.log.info("Check that txs from RPC are not sent to blockrelay connection") conn = self.nodes[0].add_outbound_p2p_connection(P2PTxInvStore(), p2p_idx=1, connection_type="block-relay-only") @@ -89,29 +92,24 @@ class P2PBlocksOnly(BitcoinTestFramework): # Bump time forward to ensure nNextInvSend timer pops self.nodes[0].setmocktime(int(time.time()) + 60) - # Calling sync_with_ping twice requires that the node calls - # `ProcessMessage` twice, and thus ensures `SendMessages` must have - # been called at least once - conn.sync_with_ping() - conn.sync_with_ping() + conn.sync_send_with_ping() assert(int(txid, 16) not in conn.get_invs()) def check_p2p_tx_violation(self, index=1): self.log.info('Check that txs from P2P are rejected and result in disconnect') input_txid = self.nodes[0].getblock(self.nodes[0].getblockhash(index), 2)['tx'][0]['txid'] - tx = create_transaction(self.nodes[0], input_txid, self.nodes[0].getnewaddress(), amount=(50 - 0.001)) - txid = tx.rehash() - tx_hex = tx.serialize().hex() + utxo_to_spend = self.miniwallet.get_utxo(txid=input_txid) + spendtx = self.miniwallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_to_spend) with self.nodes[0].assert_debug_log(['transaction sent in violation of protocol peer=0']): - self.nodes[0].p2ps[0].send_message(msg_tx(tx)) + self.nodes[0].p2ps[0].send_message(msg_tx(spendtx['tx'])) self.nodes[0].p2ps[0].wait_for_disconnect() assert_equal(self.nodes[0].getmempoolinfo()['size'], 0) # Remove the disconnected peer del self.nodes[0].p2ps[0] - return tx, txid, tx_hex + return spendtx['tx'], spendtx['txid'], spendtx['wtxid'], spendtx['hex'] if __name__ == '__main__': diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 4bee33f825..359cfb9c34 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -174,8 +174,7 @@ class FilterTest(BitcoinTestFramework): filter_peer.merkleblock_received = False filter_peer.tx_received = False self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 90) - filter_peer.sync_with_ping() - filter_peer.sync_with_ping() + filter_peer.sync_send_with_ping() assert not filter_peer.merkleblock_received assert not filter_peer.tx_received diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index c0b3c2cb12..788a81d4af 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -37,7 +37,7 @@ VALID_DATA_LIMIT = MAX_PROTOCOL_MESSAGE_LENGTH - 5 # Account for the 5-byte len class msg_unrecognized: """Nonsensical message. Modeled after similar types in test_framework.messages.""" - msgtype = b'badmsg' + msgtype = b'badmsg\x01' def __init__(self, *, str_data): self.str_data = str_data.encode() if not isinstance(str_data, bytes) else str_data @@ -104,7 +104,7 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_magic_bytes(self): self.log.info("Test message with invalid magic bytes disconnects peer") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - with self.nodes[0].assert_debug_log(['HEADER ERROR - MESSAGESTART (badmsg, 2 bytes), received ffffffff']): + with self.nodes[0].assert_debug_log(['Header error: Wrong MessageStart ffffffff received']): msg = conn.build_message(msg_unrecognized(str_data="d")) # modify magic bytes msg = b'\xff' * 4 + msg[4:] @@ -115,7 +115,7 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_checksum(self): self.log.info("Test message with invalid checksum logs an error") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - with self.nodes[0].assert_debug_log(['CHECKSUM ERROR (badmsg, 2 bytes), expected 78df0a04 was ffffffff']): + with self.nodes[0].assert_debug_log(['Header error: Wrong checksum (badmsg, 2 bytes), expected 78df0a04 was ffffffff']): msg = conn.build_message(msg_unrecognized(str_data="d")) # Checksum is after start bytes (4B), message type (12B), len (4B) cut_len = 4 + 12 + 4 @@ -130,7 +130,7 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_size(self): self.log.info("Test message with oversized payload disconnects peer") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - with self.nodes[0].assert_debug_log(['HEADER ERROR - SIZE (badmsg, 4000001 bytes)']): + with self.nodes[0].assert_debug_log(['Header error: Size too large (badmsg, 4000001 bytes)']): msg = msg_unrecognized(str_data="d" * (VALID_DATA_LIMIT + 1)) msg = conn.build_message(msg) conn.send_raw_message(msg) @@ -140,7 +140,7 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_msgtype(self): self.log.info("Test message with invalid message type logs an error") conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - with self.nodes[0].assert_debug_log(['HEADER ERROR - COMMAND']): + with self.nodes[0].assert_debug_log(['Header error: Invalid message type']): msg = msg_unrecognized(str_data="d") msg = conn.build_message(msg) # Modify msgtype diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 54891b07e1..9d32c1cb86 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -164,7 +164,7 @@ class TestP2PConn(P2PInterface): def on_wtxidrelay(self, message): self.last_wtxidrelay.append(message) - def announce_tx_and_wait_for_getdata(self, tx, timeout=60, success=True, use_wtxid=False): + def announce_tx_and_wait_for_getdata(self, tx, success=True, use_wtxid=False): if success: # sanity check assert (self.wtxidrelay and use_wtxid) or (not self.wtxidrelay and not use_wtxid) @@ -178,11 +178,11 @@ class TestP2PConn(P2PInterface): if success: if use_wtxid: - self.wait_for_getdata([wtxid], timeout) + self.wait_for_getdata([wtxid]) else: - self.wait_for_getdata([tx.sha256], timeout) + self.wait_for_getdata([tx.sha256]) else: - time.sleep(timeout) + time.sleep(5) assert not self.last_message.get("getdata") def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): @@ -604,7 +604,7 @@ class SegWitTest(BitcoinTestFramework): # Since we haven't delivered the tx yet, inv'ing the same tx from # a witness transaction ought not result in a getdata. - self.test_node.announce_tx_and_wait_for_getdata(tx, timeout=2, success=False) + self.test_node.announce_tx_and_wait_for_getdata(tx, success=False) # Delivering this transaction with witness should fail (no matter who # its from) @@ -1461,7 +1461,7 @@ class SegWitTest(BitcoinTestFramework): self.std_node.announce_tx_and_wait_for_getdata(tx3) test_transaction_acceptance(self.nodes[1], self.std_node, tx3, with_witness=True, accepted=False, reason="bad-txns-nonstandard-inputs") # Now the node will no longer ask for getdata of this transaction when advertised by same txid - self.std_node.announce_tx_and_wait_for_getdata(tx3, timeout=5, success=False) + self.std_node.announce_tx_and_wait_for_getdata(tx3, success=False) # Spending a higher version witness output is not allowed by policy, # even with fRequireStandard=false. @@ -1956,22 +1956,34 @@ class SegWitTest(BitcoinTestFramework): def test_upgrade_after_activation(self): """Test the behavior of starting up a segwit-aware node after the softfork has activated.""" - self.restart_node(2, extra_args=["-segwitheight={}".format(SEGWIT_HEIGHT)]) + # All nodes are caught up and node 2 is a pre-segwit node that will soon upgrade. + for n in range(2): + assert_equal(self.nodes[n].getblockcount(), self.nodes[2].getblockcount()) + assert softfork_active(self.nodes[n], "segwit") + assert SEGWIT_HEIGHT < self.nodes[2].getblockcount() + assert 'segwit' not in self.nodes[2].getblockchaininfo()['softforks'] + + # Restarting node 2 should result in a shutdown because the blockchain consists of + # insufficiently validated blocks per segwit consensus rules. + self.stop_node(2) + self.nodes[2].assert_start_raises_init_error( + extra_args=[f"-segwitheight={SEGWIT_HEIGHT}"], + expected_msg=f": Witness data for blocks after height {SEGWIT_HEIGHT} requires validation. Please restart with -reindex..\nPlease restart with -reindex or -reindex-chainstate to recover.", + ) + + # As directed, the user restarts the node with -reindex + self.start_node(2, extra_args=["-reindex", f"-segwitheight={SEGWIT_HEIGHT}"]) + + # With the segwit consensus rules, the node is able to validate only up to SEGWIT_HEIGHT - 1 + assert_equal(self.nodes[2].getblockcount(), SEGWIT_HEIGHT - 1) self.connect_nodes(0, 2) # We reconnect more than 100 blocks, give it plenty of time + # sync_blocks() also verifies the best block hash is the same for all nodes self.sync_blocks(timeout=240) - # Make sure that this peer thinks segwit has activated. - assert softfork_active(self.nodes[2], 'segwit') - - # Make sure this peer's blocks match those of node0. - height = self.nodes[2].getblockcount() - while height >= 0: - block_hash = self.nodes[2].getblockhash(height) - assert_equal(block_hash, self.nodes[0].getblockhash(height)) - assert_equal(self.nodes[0].getblock(block_hash), self.nodes[2].getblock(block_hash)) - height -= 1 + # The upgraded node should now have segwit activated + assert softfork_active(self.nodes[2], "segwit") @subtest # type: ignore def test_witness_sigops(self): diff --git a/test/functional/rpc_addresses_deprecation.py b/test/functional/rpc_addresses_deprecation.py index bc0559f3b5..bc0559f3b5 100644..100755 --- a/test/functional/rpc_addresses_deprecation.py +++ b/test/functional/rpc_addresses_deprecation.py diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index ac53a280b4..00324347ed 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -275,7 +275,7 @@ class BlockchainTest(BitcoinTestFramework): assert 'muhash' in res6 assert(res['hash_serialized_2'] != res6['muhash']) - # muhash should not be included in gettxoutset unless requested. + # muhash should not be returned unless requested. for r in [res, res2, res3, res4, res5]: assert 'muhash' not in r diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index e65787ce08..dc469ba552 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -41,7 +41,7 @@ class DumptxoutsetTest(BitcoinTestFramework): digest = hashlib.sha256(f.read()).hexdigest() # UTXO snapshot hash should be deterministic based on mocked time. assert_equal( - digest, 'be032e5f248264ba08e11099ac09dbd001f6f87ffc68bf0f87043d8146d50664') + digest, '7ae82c986fa5445678d2a21453bb1c86d39e47af13da137640c2b1cf8093691c') # Specifying a path to an existing file will fail. assert_raises_rpc_error( diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 5129ecb895..0cce5493fc 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -5,6 +5,8 @@ """Test the fundrawtransaction RPC.""" from decimal import Decimal +from itertools import product + from test_framework.descriptors import descsum_create from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -723,17 +725,16 @@ class RawTransactionsTest(BitcoinTestFramework): result2 = node.fundrawtransaction(rawtx, {"feeRate": 2 * self.min_relay_tx_fee}) result3 = node.fundrawtransaction(rawtx, {"fee_rate": 10 * btc_kvb_to_sat_vb * self.min_relay_tx_fee}) result4 = node.fundrawtransaction(rawtx, {"feeRate": str(10 * self.min_relay_tx_fee)}) - # Test that funding non-standard "zero-fee" transactions is valid. - result5 = self.nodes[3].fundrawtransaction(rawtx, {"fee_rate": 0}) - result6 = self.nodes[3].fundrawtransaction(rawtx, {"feeRate": 0}) result_fee_rate = result['fee'] * 1000 / count_bytes(result['hex']) - assert_fee_amount(result1['fee'], count_bytes(result2['hex']), 2 * result_fee_rate) + assert_fee_amount(result1['fee'], count_bytes(result1['hex']), 2 * result_fee_rate) assert_fee_amount(result2['fee'], count_bytes(result2['hex']), 2 * result_fee_rate) assert_fee_amount(result3['fee'], count_bytes(result3['hex']), 10 * result_fee_rate) - assert_fee_amount(result4['fee'], count_bytes(result3['hex']), 10 * result_fee_rate) - assert_fee_amount(result5['fee'], count_bytes(result5['hex']), 0) - assert_fee_amount(result6['fee'], count_bytes(result6['hex']), 0) + assert_fee_amount(result4['fee'], count_bytes(result4['hex']), 10 * result_fee_rate) + + # Test that funding non-standard "zero-fee" transactions is valid. + for param, zero_value in product(["fee_rate", "feeRate"], [0, 0.000, 0.00000000, "0", "0.000", "0.00000000"]): + assert_equal(self.nodes[3].fundrawtransaction(rawtx, {param: zero_value})["fee"], 0) # With no arguments passed, expect fee of 141 satoshis. assert_approx(node.fundrawtransaction(rawtx)["fee"], vexp=0.00000141, vspan=0.00000001) @@ -767,11 +768,16 @@ class RawTransactionsTest(BitcoinTestFramework): node.fundrawtransaction, rawtx, {param: -1, "add_inputs": True}) assert_raises_rpc_error(-3, "Amount is not a number or string", node.fundrawtransaction, rawtx, {param: {"foo": "bar"}, "add_inputs": True}) + # Test fee rate values that don't pass fixed-point parsing checks. + for invalid_value in ["", 0.000000001, 1e-09, 1.111111111, 1111111111111111, "31.999999999999999999999"]: + assert_raises_rpc_error(-3, "Invalid amount", node.fundrawtransaction, rawtx, {param: invalid_value, "add_inputs": True}) + # Test fee_rate values that cannot be represented in sat/vB. + for invalid_value in [0.0001, 0.00000001, 0.00099999, 31.99999999, "0.0001", "0.00000001", "0.00099999", "31.99999999"]: assert_raises_rpc_error(-3, "Invalid amount", - node.fundrawtransaction, rawtx, {param: "", "add_inputs": True}) + node.fundrawtransaction, rawtx, {"fee_rate": invalid_value, "add_inputs": True}) self.log.info("Test min fee rate checks are bypassed with fundrawtxn, e.g. a fee_rate under 1 sat/vB is allowed") - node.fundrawtransaction(rawtx, {"fee_rate": 0.99999999, "add_inputs": True}) + node.fundrawtransaction(rawtx, {"fee_rate": 0.999, "add_inputs": True}) node.fundrawtransaction(rawtx, {"feeRate": 0.00000999, "add_inputs": True}) self.log.info("- raises RPC error if both feeRate and fee_rate are passed") diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index 1398d1237f..52c8fa883d 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -61,30 +61,30 @@ class RpcMiscTest(BitcoinTestFramework): node.logging(include=['qt']) assert_equal(node.logging()['qt'], True) + self.log.info("test echoipc (testing spawned process in multiprocess build)") + assert_equal(node.echoipc("hello"), "hello") + self.log.info("test getindexinfo") # Without any indices running the RPC returns an empty object assert_equal(node.getindexinfo(), {}) # Restart the node with indices and wait for them to sync - self.restart_node(0, ["-txindex", "-blockfilterindex"]) + self.restart_node(0, ["-txindex", "-blockfilterindex", "-coinstatsindex"]) self.wait_until(lambda: all(i["synced"] for i in node.getindexinfo().values())) # Returns a list of all running indices by default + values = {"synced": True, "best_block_height": 200} assert_equal( node.getindexinfo(), { - "txindex": {"synced": True, "best_block_height": 200}, - "basic block filter index": {"synced": True, "best_block_height": 200} + "txindex": values, + "basic block filter index": values, + "coinstatsindex": values, } ) - # Specifying an index by name returns only the status of that index - assert_equal( - node.getindexinfo("txindex"), - { - "txindex": {"synced": True, "best_block_height": 200}, - } - ) + for i in {"txindex", "basic block filter index", "coinstatsindex"}: + assert_equal(node.getindexinfo(i), {i: values}) # Specifying an unknown index name returns an empty result assert_equal(node.getindexinfo("foo"), {}) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 079a3bd3ba..c6e96e4a92 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -6,6 +6,8 @@ """ from decimal import Decimal +from itertools import product + from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -193,14 +195,14 @@ class PSBTTest(BitcoinTestFramework): assert_approx(res2["fee"], 0.055, 0.005) self.log.info("Test min fee rate checks with walletcreatefundedpsbt are bypassed, e.g. a fee_rate under 1 sat/vB is allowed") - res3 = self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {"fee_rate": "0.99999999", "add_inputs": True}) + res3 = self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {"fee_rate": "0.999", "add_inputs": True}) assert_approx(res3["fee"], 0.00000381, 0.0000001) res4 = self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {"feeRate": 0.00000999, "add_inputs": True}) assert_approx(res4["fee"], 0.00000381, 0.0000001) self.log.info("Test min fee rate checks with walletcreatefundedpsbt are bypassed and that funding non-standard 'zero-fee' transactions is valid") - for param in ["fee_rate", "feeRate"]: - assert_equal(self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {param: 0, "add_inputs": True})["fee"], 0) + for param, zero_value in product(["fee_rate", "feeRate"], [0, 0.000, 0.00000000, "0", "0.000", "0.00000000"]): + assert_equal(0, self.nodes[1].walletcreatefundedpsbt(inputs, outputs, 0, {param: zero_value, "add_inputs": True})["fee"]) self.log.info("Test invalid fee rate settings") for param, value in {("fee_rate", 100000), ("feeRate", 1)}: @@ -210,8 +212,14 @@ class PSBTTest(BitcoinTestFramework): self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {param: -1, "add_inputs": True}) assert_raises_rpc_error(-3, "Amount is not a number or string", self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {param: {"foo": "bar"}, "add_inputs": True}) + # Test fee rate values that don't pass fixed-point parsing checks. + for invalid_value in ["", 0.000000001, 1e-09, 1.111111111, 1111111111111111, "31.999999999999999999999"]: + assert_raises_rpc_error(-3, "Invalid amount", + self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {param: invalid_value, "add_inputs": True}) + # Test fee_rate values that cannot be represented in sat/vB. + for invalid_value in [0.0001, 0.00000001, 0.00099999, 31.99999999, "0.0001", "0.00000001", "0.00099999", "31.99999999"]: assert_raises_rpc_error(-3, "Invalid amount", - self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {param: "", "add_inputs": True}) + self.nodes[1].walletcreatefundedpsbt, inputs, outputs, 0, {"fee_rate": invalid_value, "add_inputs": True}) self.log.info("- raises RPC error if both feeRate and fee_rate are passed") assert_raises_rpc_error(-8, "Cannot specify both fee_rate (sat/vB) and feeRate (BTC/kvB)", diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index e691b63df6..d08e025178 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -115,7 +115,7 @@ def script_BIP34_coinbase_height(height): return CScript([CScriptNum(height)]) -def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0): +def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValue=50): """Create a coinbase transaction. If pubkey is passed in, the coinbase output will be a P2PK output; @@ -126,10 +126,11 @@ def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0): coinbase = CTransaction() coinbase.vin.append(CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff)) coinbaseoutput = CTxOut() - coinbaseoutput.nValue = 50 * COIN - halvings = int(height / 150) # regtest - coinbaseoutput.nValue >>= halvings - coinbaseoutput.nValue += fees + coinbaseoutput.nValue = nValue * COIN + if nValue == 50: + halvings = int(height / 150) # regtest + coinbaseoutput.nValue >>= halvings + coinbaseoutput.nValue += fees if pubkey is not None: coinbaseoutput.scriptPubKey = CScript([pubkey, OP_CHECKSIG]) else: diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 05099f3339..cc80b543cd 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -539,8 +539,16 @@ class P2PInterface(P2PConnection): self.send_message(message) self.sync_with_ping(timeout=timeout) - # Sync up with the node + def sync_send_with_ping(self, timeout=60): + """Ensure SendMessages is called on this connection""" + # Calling sync_with_ping twice requires that the node calls + # `ProcessMessage` twice, and thus ensures `SendMessages` must have + # been called at least once + self.sync_with_ping() + self.sync_with_ping() + def sync_with_ping(self, timeout=60): + """Ensure ProcessMessages is called on this connection""" self.send_message(msg_ping(nonce=self.ping_counter)) def test_function(): diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 02eb10b5a4..a89a26caea 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -739,11 +739,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # block in the cache does not age too much (have an old tip age). # This is needed so that we are out of IBD when the test starts, # see the tip age check in IsInitialBlockDownload(). - gen_addresses = [k.address for k in TestNode.PRIV_KEYS] + [ADDRESS_BCRT1_P2WSH_OP_TRUE] + gen_addresses = [k.address for k in TestNode.PRIV_KEYS][:3] + [ADDRESS_BCRT1_P2WSH_OP_TRUE] + assert_equal(len(gen_addresses), 4) for i in range(8): cache_node.generatetoaddress( nblocks=25 if i != 7 else 24, - address=gen_addresses[i % 4], + address=gen_addresses[i % len(gen_addresses)], ) assert_equal(cache_node.getblockchaininfo()["blocks"], 199) @@ -757,7 +758,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): os.rmdir(cache_path('wallets')) # Remove empty wallets dir for entry in os.listdir(cache_path()): - if entry not in ['chainstate', 'blocks']: # Only keep chainstate and blocks folder + if entry not in ['chainstate', 'blocks', 'indexes']: # Only indexes, chainstate and blocks folders os.remove(cache_path(entry)) for i in range(self.num_nodes): diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index ce9c1bc024..c17c16f797 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -491,6 +491,7 @@ class TestNode(): self.start(extra_args, stdout=log_stdout, stderr=log_stderr, *args, **kwargs) ret = self.process.wait(timeout=self.rpc_timeout) self.log.debug(self._node_msg(f'bitcoind exited with status {ret} during initialization')) + assert ret != 0 # Exit code must indicate failure self.running = False self.process = None # Check stderr for expected message diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index a906a21dd0..395b50c4d8 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -17,6 +17,7 @@ from test_framework.messages import ( from test_framework.script import ( CScript, OP_TRUE, + OP_NOP, ) from test_framework.util import ( assert_equal, @@ -26,24 +27,32 @@ from test_framework.util import ( class MiniWallet: - def __init__(self, test_node): + def __init__(self, test_node, *, raw_script=False): self._test_node = test_node self._utxos = [] - self._address = ADDRESS_BCRT1_P2WSH_OP_TRUE - self._scriptPubKey = hex_str_to_bytes(self._test_node.validateaddress(self._address)['scriptPubKey']) + if raw_script: + self._address = None + self._scriptPubKey = bytes(CScript([OP_TRUE])) + else: + self._address = ADDRESS_BCRT1_P2WSH_OP_TRUE + self._scriptPubKey = hex_str_to_bytes(self._test_node.validateaddress(self._address)['scriptPubKey']) def scan_blocks(self, *, start=1, num): """Scan the blocks for self._address outputs and add them to self._utxos""" for i in range(start, start + num): block = self._test_node.getblock(blockhash=self._test_node.getblockhash(i), verbosity=2) for tx in block['tx']: - for out in tx['vout']: - if out['scriptPubKey']['hex'] == self._scriptPubKey.hex(): - self._utxos.append({'txid': tx['txid'], 'vout': out['n'], 'value': out['value']}) + self.scan_tx(tx) + + def scan_tx(self, tx): + """Scan the tx for self._scriptPubKey outputs and add them to self._utxos""" + for out in tx['vout']: + if out['scriptPubKey']['hex'] == self._scriptPubKey.hex(): + self._utxos.append({'txid': tx['txid'], 'vout': out['n'], 'value': out['value']}) def generate(self, num_blocks): """Generate blocks with coinbase outputs to the internal address, and append the outputs to the internal list""" - blocks = self._test_node.generatetoaddress(num_blocks, self._address) + blocks = self._test_node.generatetodescriptor(num_blocks, f'raw({self._scriptPubKey.hex()})') for b in blocks: cb_tx = self._test_node.getblock(blockhash=b, verbosity=2)['tx'][0] self._utxos.append({'txid': cb_tx['txid'], 'vout': 0, 'value': cb_tx['vout'][0]['value']}) @@ -69,6 +78,12 @@ class MiniWallet: def send_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node, utxo_to_spend=None): """Create and send a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + tx = self.create_self_transfer(fee_rate=fee_rate, from_node=from_node, utxo_to_spend=utxo_to_spend) + self.sendrawtransaction(from_node=from_node, tx_hex=tx['hex']) + return tx + + def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node, utxo_to_spend=None, mempool_valid=True): + """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" self._utxos = sorted(self._utxos, key=lambda k: k['value']) utxo_to_spend = utxo_to_spend or self._utxos.pop() # Pick the largest utxo (if none provided) and hope it covers the fee vsize = Decimal(96) @@ -79,13 +94,21 @@ class MiniWallet: tx = CTransaction() tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']))] tx.vout = [CTxOut(int(send_value * COIN), self._scriptPubKey)] - tx.wit.vtxinwit = [CTxInWitness()] - tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + if not self._address: + # raw script + tx.vin[0].scriptSig = CScript([OP_NOP] * 35) # pad to identical size + else: + tx.wit.vtxinwit = [CTxInWitness()] + tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] tx_hex = tx.serialize().hex() tx_info = from_node.testmempoolaccept([tx_hex])[0] - self._utxos.append({'txid': tx_info['txid'], 'vout': 0, 'value': send_value}) + assert_equal(mempool_valid, tx_info['allowed']) + if mempool_valid: + assert_equal(tx_info['vsize'], vsize) + assert_equal(tx_info['fees']['base'], fee) + return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex, 'tx': tx} + + def sendrawtransaction(self, *, from_node, tx_hex): from_node.sendrawtransaction(tx_hex) - assert_equal(tx_info['vsize'], vsize) - assert_equal(tx_info['fees']['base'], fee) - return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex} + self.scan_tx(from_node.decoderawtransaction(tx_hex)) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index bd58f2cd51..00527e78f1 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -281,6 +281,7 @@ BASE_SCRIPTS = [ 'rpc_scantxoutset.py', 'feature_logging.py', 'feature_anchors.py', + 'feature_coinstatsindex.py', 'p2p_node_network_limited.py', 'p2p_permissions.py', 'feature_blocksdir.py', diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 46eecae611..4a1d25bbc5 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -15,6 +15,7 @@ from test_framework.util import ( ) from test_framework.wallet_util import test_address +NOT_A_NUMBER_OR_STRING = "Amount is not a number or string" OUT_OF_RANGE = "Amount out of range" @@ -264,12 +265,25 @@ class WalletTest(BitcoinTestFramework): # Test setting explicit fee rate just below the minimum. self.log.info("Test sendmany raises 'fee rate too low' if fee_rate of 0.99999999 is passed") assert_raises_rpc_error(-6, "Fee rate (0.999 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)", - self.nodes[2].sendmany, amounts={address: 10}, fee_rate=0.99999999) - - self.log.info("Test sendmany raises if fee_rate of 0 or -1 is passed") - assert_raises_rpc_error(-6, "Fee rate (0.000 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)", - self.nodes[2].sendmany, amounts={address: 10}, fee_rate=0) + self.nodes[2].sendmany, amounts={address: 10}, fee_rate=0.999) + + self.log.info("Test sendmany raises if an invalid fee_rate is passed") + # Test fee_rate with zero values. + msg = "Fee rate (0.000 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)" + for zero_value in [0, 0.000, 0.00000000, "0", "0.000", "0.00000000"]: + assert_raises_rpc_error(-6, msg, self.nodes[2].sendmany, amounts={address: 1}, fee_rate=zero_value) + msg = "Invalid amount" + # Test fee_rate values that don't pass fixed-point parsing checks. + for invalid_value in ["", 0.000000001, 1e-09, 1.111111111, 1111111111111111, "31.999999999999999999999"]: + assert_raises_rpc_error(-3, msg, self.nodes[2].sendmany, amounts={address: 1.0}, fee_rate=invalid_value) + # Test fee_rate values that cannot be represented in sat/vB. + for invalid_value in [0.0001, 0.00000001, 0.00099999, 31.99999999, "0.0001", "0.00000001", "0.00099999", "31.99999999"]: + assert_raises_rpc_error(-3, msg, self.nodes[2].sendmany, amounts={address: 10}, fee_rate=invalid_value) + # Test fee_rate out of range (negative number). assert_raises_rpc_error(-3, OUT_OF_RANGE, self.nodes[2].sendmany, amounts={address: 10}, fee_rate=-1) + # Test type error. + for invalid_value in [True, {"foo": "bar"}]: + assert_raises_rpc_error(-3, NOT_A_NUMBER_OR_STRING, self.nodes[2].sendmany, amounts={address: 10}, fee_rate=invalid_value) self.log.info("Test sendmany raises if an invalid conf_target or estimate_mode is passed") for target, mode in product([-1, 0, 1009], ["economical", "conservative"]): @@ -447,12 +461,25 @@ class WalletTest(BitcoinTestFramework): # Test setting explicit fee rate just below the minimum. self.log.info("Test sendtoaddress raises 'fee rate too low' if fee_rate of 0.99999999 is passed") assert_raises_rpc_error(-6, "Fee rate (0.999 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)", - self.nodes[2].sendtoaddress, address=address, amount=1, fee_rate=0.99999999) - - self.log.info("Test sendtoaddress raises if fee_rate of 0 or -1 is passed") - assert_raises_rpc_error(-6, "Fee rate (0.000 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)", - self.nodes[2].sendtoaddress, address=address, amount=10, fee_rate=0) + self.nodes[2].sendtoaddress, address=address, amount=1, fee_rate=0.999) + + self.log.info("Test sendtoaddress raises if an invalid fee_rate is passed") + # Test fee_rate with zero values. + msg = "Fee rate (0.000 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)" + for zero_value in [0, 0.000, 0.00000000, "0", "0.000", "0.00000000"]: + assert_raises_rpc_error(-6, msg, self.nodes[2].sendtoaddress, address=address, amount=1, fee_rate=zero_value) + msg = "Invalid amount" + # Test fee_rate values that don't pass fixed-point parsing checks. + for invalid_value in ["", 0.000000001, 1e-09, 1.111111111, 1111111111111111, "31.999999999999999999999"]: + assert_raises_rpc_error(-3, msg, self.nodes[2].sendtoaddress, address=address, amount=1.0, fee_rate=invalid_value) + # Test fee_rate values that cannot be represented in sat/vB. + for invalid_value in [0.0001, 0.00000001, 0.00099999, 31.99999999, "0.0001", "0.00000001", "0.00099999", "31.99999999"]: + assert_raises_rpc_error(-3, msg, self.nodes[2].sendtoaddress, address=address, amount=10, fee_rate=invalid_value) + # Test fee_rate out of range (negative number). assert_raises_rpc_error(-3, OUT_OF_RANGE, self.nodes[2].sendtoaddress, address=address, amount=1.0, fee_rate=-1) + # Test type error. + for invalid_value in [True, {"foo": "bar"}]: + assert_raises_rpc_error(-3, NOT_A_NUMBER_OR_STRING, self.nodes[2].sendtoaddress, address=address, amount=1.0, fee_rate=invalid_value) self.log.info("Test sendtoaddress raises if an invalid conf_target or estimate_mode is passed") for target, mode in product([-1, 0, 1009], ["economical", "conservative"]): diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 0d1b6c54ce..ff5070c1fa 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -110,13 +110,24 @@ class BumpFeeTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Insufficient total fee 0.00000141", rbf_node.bumpfee, rbfid, {"fee_rate": INSUFFICIENT}) self.log.info("Test invalid fee rate settings") - assert_raises_rpc_error(-8, "Insufficient total fee 0.00", rbf_node.bumpfee, rbfid, {"fee_rate": 0}) assert_raises_rpc_error(-4, "Specified or calculated fee 0.141 is too high (cannot be higher than -maxtxfee 0.10", rbf_node.bumpfee, rbfid, {"fee_rate": TOO_HIGH}) + # Test fee_rate with zero values. + msg = "Insufficient total fee 0.00" + for zero_value in [0, 0.000, 0.00000000, "0", "0.000", "0.00000000"]: + assert_raises_rpc_error(-8, msg, rbf_node.bumpfee, rbfid, {"fee_rate": zero_value}) + msg = "Invalid amount" + # Test fee_rate values that don't pass fixed-point parsing checks. + for invalid_value in ["", 0.000000001, 1e-09, 1.111111111, 1111111111111111, "31.999999999999999999999"]: + assert_raises_rpc_error(-3, msg, rbf_node.bumpfee, rbfid, {"fee_rate": invalid_value}) + # Test fee_rate values that cannot be represented in sat/vB. + for invalid_value in [0.0001, 0.00000001, 0.00099999, 31.99999999, "0.0001", "0.00000001", "0.00099999", "31.99999999"]: + assert_raises_rpc_error(-3, msg, rbf_node.bumpfee, rbfid, {"fee_rate": invalid_value}) + # Test fee_rate out of range (negative number). assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate": -1}) + # Test type error. for value in [{"foo": "bar"}, True]: assert_raises_rpc_error(-3, "Amount is not a number or string", rbf_node.bumpfee, rbfid, {"fee_rate": value}) - assert_raises_rpc_error(-3, "Invalid amount", rbf_node.bumpfee, rbfid, {"fee_rate": ""}) self.log.info("Test explicit fee rate raises RPC error if both fee_rate and conf_target are passed") assert_raises_rpc_error(-8, "Cannot specify both conf_target and fee_rate. Please provide either a confirmation " diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index ed62ef0e9d..0a3dd56620 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -459,6 +459,77 @@ class ImportDescriptorsTest(BitcoinTestFramework): assert_equal(tx_signed_2['complete'], True) self.nodes[1].sendrawtransaction(tx_signed_2['hex']) + self.log.info("We can create and use a huge multisig under P2WSH") + self.nodes[1].createwallet(wallet_name='wmulti_priv_big', blank=True, descriptors=True) + wmulti_priv_big = self.nodes[1].get_wallet_rpc('wmulti_priv_big') + xkey = "tprv8ZgxMBicQKsPeZSeYx7VXDDTs3XrTcmZQpRLbAeSQFCQGgKwR4gKpcxHaKdoTNHniv4EPDJNdzA3KxRrrBHcAgth8fU5X4oCndkkxk39iAt/*" + xkey_int = "tprv8ZgxMBicQKsPeZSeYx7VXDDTs3XrTcmZQpRLbAeSQFCQGgKwR4gKpcxHaKdoTNHniv4EPDJNdzA3KxRrrBHcAgth8fU5X4oCndkkxk39iAt/1/*" + res = wmulti_priv_big.importdescriptors([ + { + "desc": descsum_create(f"wsh(sortedmulti(20,{(xkey + ',') * 19}{xkey}))"), + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }, + { + "desc": descsum_create(f"wsh(sortedmulti(20,{(xkey_int + ',') * 19}{xkey_int}))"), + "active": True, + "internal": True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }]) + assert_equal(res[0]['success'], True) + assert_equal(res[1]['success'], True) + + addr = wmulti_priv_big.getnewaddress() + w0.sendtoaddress(addr, 10) + self.nodes[0].generate(1) + self.sync_all() + # It is standard and would relay. + txid = wmulti_priv_big.sendtoaddress(w0.getnewaddress(), 9.999) + decoded = wmulti_priv_big.decoderawtransaction(wmulti_priv_big.gettransaction(txid)['hex']) + # 20 sigs + dummy + witness script + assert_equal(len(decoded['vin'][0]['txinwitness']), 22) + + + self.log.info("Under P2SH, multisig are standard with up to 15 " + "compressed keys") + self.nodes[1].createwallet(wallet_name='multi_priv_big_legacy', + blank=True, descriptors=True) + multi_priv_big = self.nodes[1].get_wallet_rpc('multi_priv_big_legacy') + res = multi_priv_big.importdescriptors([ + { + "desc": descsum_create(f"sh(multi(15,{(xkey + ',') * 14}{xkey}))"), + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }, + { + "desc": descsum_create(f"sh(multi(15,{(xkey_int + ',') * 14}{xkey_int}))"), + "active": True, + "internal": True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }]) + assert_equal(res[0]['success'], True) + assert_equal(res[1]['success'], True) + + addr = multi_priv_big.getnewaddress("", "legacy") + w0.sendtoaddress(addr, 10) + self.nodes[0].generate(6) + self.sync_all() + # It is standard and would relay. + txid = multi_priv_big.sendtoaddress(w0.getnewaddress(), 10, "", "", + True) + decoded = multi_priv_big.decoderawtransaction( + multi_priv_big.gettransaction(txid)['hex'] + ) + + self.log.info("Combo descriptors cannot be active") self.test_importdesc({"desc": descsum_create("combo(tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*)"), "active": True, diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 53553dcd80..e061fa952c 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -343,22 +343,41 @@ class WalletSendTest(BitcoinTestFramework): self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_conf_target=0.1, arg_estimate_mode=mode, expect_error=(-8, msg)) assert_raises_rpc_error(-8, msg, w0.send, {w1.getnewaddress(): 1}, 0.1, mode) - for mode in ["economical", "conservative", "btc/kb", "sat/b"]: - self.log.debug("{}".format(mode)) - for k, v in {"string": "true", "object": {"foo": "bar"}}.items(): + for mode in ["economical", "conservative"]: + for k, v in {"string": "true", "bool": True, "object": {"foo": "bar"}}.items(): self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=v, estimate_mode=mode, - expect_error=(-3, "Expected type number for conf_target, got {}".format(k))) + expect_error=(-3, f"Expected type number for conf_target, got {k}")) - # Test setting explicit fee rate just below the minimum and at zero. + # Test setting explicit fee rate just below the minimum of 1 sat/vB. self.log.info("Explicit fee rate raises RPC error 'fee rate too low' if fee_rate of 0.99999999 is passed") - self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=0.99999999, - expect_error=(-4, "Fee rate (0.999 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)")) - self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=0.99999999, - expect_error=(-4, "Fee rate (0.999 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)")) - self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=0, - expect_error=(-4, "Fee rate (0.000 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)")) - self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=0, - expect_error=(-4, "Fee rate (0.000 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)")) + msg = "Fee rate (0.999 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)" + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=0.999, expect_error=(-4, msg)) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=0.999, expect_error=(-4, msg)) + + self.log.info("Explicit fee rate raises if invalid fee_rate is passed") + # Test fee_rate with zero values. + msg = "Fee rate (0.000 sat/vB) is lower than the minimum fee rate setting (1.000 sat/vB)" + for zero_value in [0, 0.000, 0.00000000, "0", "0.000", "0.00000000"]: + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=zero_value, expect_error=(-4, msg)) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=zero_value, expect_error=(-4, msg)) + msg = "Invalid amount" + # Test fee_rate values that don't pass fixed-point parsing checks. + for invalid_value in ["", 0.000000001, 1e-09, 1.111111111, 1111111111111111, "31.999999999999999999999"]: + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=invalid_value, expect_error=(-3, msg)) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=invalid_value, expect_error=(-3, msg)) + # Test fee_rate values that cannot be represented in sat/vB. + for invalid_value in [0.0001, 0.00000001, 0.00099999, 31.99999999, "0.0001", "0.00000001", "0.00099999", "31.99999999"]: + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=invalid_value, expect_error=(-3, msg)) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=invalid_value, expect_error=(-3, msg)) + # Test fee_rate out of range (negative number). + msg = "Amount out of range" + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=-1, expect_error=(-3, msg)) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=-1, expect_error=(-3, msg)) + # Test type error. + msg = "Amount is not a number or string" + for invalid_value in [True, {"foo": "bar"}]: + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, fee_rate=invalid_value, expect_error=(-3, msg)) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_fee_rate=invalid_value, expect_error=(-3, msg)) # TODO: Return hex if fee rate is below -maxmempool # res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=0.1, estimate_mode="sat/b", add_to_wallet=False) diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index ad2333a808..354e14f361 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -14,6 +14,7 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "node/blockstorage -> validation -> node/blockstorage" "index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex" "index/base -> validation -> index/blockfilterindex -> index/base" + "index/coinstatsindex -> node/coinstats -> index/coinstatsindex" "policy/fees -> txmempool -> policy/fees" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" "qt/bitcoingui -> qt/walletframe -> qt/bitcoingui" diff --git a/test/lint/lint-filenames.sh b/test/lint/lint-filenames.sh deleted file mode 100755 index 3f7491cd2b..0000000000 --- a/test/lint/lint-filenames.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure only lowercase alphanumerics (a-z0-9), underscores (_), -# hyphens (-) and dots (.) are used in source code filenames. - -export LC_ALL=C - -EXIT_CODE=0 -OUTPUT=$(git ls-files --full-name -- "*.[cC][pP][pP]" "*.[hH]" "*.[pP][yY]" "*.[sS][hH]" | \ - grep -vE '^[a-z0-9_./-]+$' | \ - grep -vE '^src/(secp256k1/|univalue/|test/fuzz/FuzzedDataProvider.h)') - -if [[ ${OUTPUT} != "" ]]; then - echo "Use only lowercase alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.)" - echo "in source code filenames:" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-files.py b/test/lint/lint-files.py new file mode 100755 index 0000000000..400921e5f3 --- /dev/null +++ b/test/lint/lint-files.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +This checks that all files in the repository have correct filenames and permissions +""" + +import os +import re +import sys +from subprocess import check_output +from typing import Optional, NoReturn + +CMD_ALL_FILES = "git ls-files -z --full-name" +CMD_SOURCE_FILES = 'git ls-files -z --full-name -- "*.[cC][pP][pP]" "*.[hH]" "*.[pP][yY]" "*.[sS][hH]"' +CMD_SHEBANG_FILES = "git grep --full-name --line-number -I '^#!'" +ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$" +ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$" +ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = ( + "^src/(secp256k1/|univalue/|test/fuzz/FuzzedDataProvider.h)" +) +ALLOWED_PERMISSION_NON_EXECUTABLES = 644 +ALLOWED_PERMISSION_EXECUTABLES = 755 +ALLOWED_EXECUTABLE_SHEBANG = { + "py": [b"#!/usr/bin/env python3"], + "sh": [b"#!/usr/bin/env bash", b"#!/bin/sh"], +} + + +class FileMeta(object): + def __init__(self, file_path: str): + self.file_path = file_path + + @property + def extension(self) -> Optional[str]: + """ + Returns the file extension for a given filename string. + eg: + 'ci/lint_run_all.sh' -> 'sh' + 'ci/retry/retry' -> None + 'contrib/devtools/split-debug.sh.in' -> 'in' + """ + return str(os.path.splitext(self.file_path)[1].strip(".") or None) + + @property + def full_extension(self) -> Optional[str]: + """ + Returns the full file extension for a given filename string. + eg: + 'ci/lint_run_all.sh' -> 'sh' + 'ci/retry/retry' -> None + 'contrib/devtools/split-debug.sh.in' -> 'sh.in' + """ + filename_parts = self.file_path.split(os.extsep, 1) + try: + return filename_parts[1] + except IndexError: + return None + + @property + def permissions(self) -> int: + """ + Returns the octal file permission of the file + """ + return int(oct(os.stat(self.file_path).st_mode)[-3:]) + + +def check_all_filenames() -> int: + """ + Checks every file in the repository against an allowed regexp to make sure only lowercase or uppercase + alphanumerics (a-zA-Z0-9), underscores (_), hyphens (-), at (@) and dots (.) are used in repository filenames. + """ + filenames = check_output(CMD_ALL_FILES, shell=True).decode("utf8").rstrip("\0").split("\0") + filename_regex = re.compile(ALLOWED_FILENAME_REGEXP) + failed_tests = 0 + for filename in filenames: + if not filename_regex.match(filename): + print( + f"""File {repr(filename)} does not not match the allowed filename regexp ('{ALLOWED_FILENAME_REGEXP}').""" + ) + failed_tests += 1 + return failed_tests + + +def check_source_filenames() -> int: + """ + Checks only source files (*.cpp, *.h, *.py, *.sh) against a stricter allowed regexp to make sure only lowercase + alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.) are used in source code filenames. + + Additionally there is an exception regexp for directories or files which are excepted from matching this regexp. + """ + filenames = check_output(CMD_SOURCE_FILES, shell=True).decode("utf8").rstrip("\0").split("\0") + filename_regex = re.compile(ALLOWED_SOURCE_FILENAME_REGEXP) + filename_exception_regex = re.compile(ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP) + failed_tests = 0 + for filename in filenames: + if not filename_regex.match(filename) and not filename_exception_regex.match(filename): + print( + f"""File {repr(filename)} does not not match the allowed source filename regexp ('{ALLOWED_SOURCE_FILENAME_REGEXP}'), or the exception regexp ({ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP}).""" + ) + failed_tests += 1 + return failed_tests + + +def check_all_file_permissions() -> int: + """ + Checks all files in the repository match an allowed executable or non-executable file permission octal. + + Additionally checks that for executable files, the file contains a shebang line + """ + filenames = check_output(CMD_ALL_FILES, shell=True).decode("utf8").rstrip("\0").split("\0") + failed_tests = 0 + for filename in filenames: + file_meta = FileMeta(filename) + if file_meta.permissions == ALLOWED_PERMISSION_EXECUTABLES: + with open(filename, "rb") as f: + shebang = f.readline().rstrip(b"\n") + + # For any file with executable permissions the first line must contain a shebang + if not shebang.startswith(b"#!"): + print( + f"""File "{filename}" has permission {ALLOWED_PERMISSION_EXECUTABLES} (executable) and is thus expected to contain a shebang '#!'. Add shebang or do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES} {filename}" to make it non-executable.""" + ) + failed_tests += 1 + + # For certain file extensions that have been defined, we also check that the shebang conforms to a specific + # allowable set of shebangs + if file_meta.extension in ALLOWED_EXECUTABLE_SHEBANG.keys(): + if shebang not in ALLOWED_EXECUTABLE_SHEBANG[file_meta.extension]: + print( + f"""File "{filename}" is missing expected shebang """ + + " or ".join( + [ + x.decode("utf-8") + for x in ALLOWED_EXECUTABLE_SHEBANG[file_meta.extension] + ] + ) + ) + failed_tests += 1 + + elif file_meta.permissions == ALLOWED_PERMISSION_NON_EXECUTABLES: + continue + else: + print( + f"""File "{filename}" has unexpected permission {file_meta.permissions}. Do "chmod {ALLOWED_PERMISSION_NON_EXECUTABLES} {filename}" (if non-executable) or "chmod {ALLOWED_PERMISSION_EXECUTABLES} {filename}" (if executable).""" + ) + failed_tests += 1 + + return failed_tests + + +def check_shebang_file_permissions() -> int: + """ + Checks every file that contains a shebang line to ensure it has an executable permission + """ + filenames = check_output(CMD_SHEBANG_FILES, shell=True).decode("utf8").strip().split("\n") + + # The git grep command we use returns files which contain a shebang on any line within the file + # so we need to filter the list to only files with the shebang on the first line + filenames = [filename.split(":1:")[0] for filename in filenames if ":1:" in filename] + + failed_tests = 0 + for filename in filenames: + file_meta = FileMeta(filename) + if file_meta.permissions != ALLOWED_PERMISSION_EXECUTABLES: + # These file types are typically expected to be sourced and not executed directly + if file_meta.full_extension in ["bash", "init", "openrc", "sh.in"]: + continue + + # *.py files which don't contain an `if __name__ == '__main__'` are not expected to be executed directly + if file_meta.extension == "py": + with open(filename, "r", encoding="utf8") as f: + file_data = f.read() + if not re.search("""if __name__ == ['"]__main__['"]:""", file_data): + continue + + print( + f"""File "{filename}" contains a shebang line, but has the file permission {file_meta.permissions} instead of the expected executable permission {ALLOWED_PERMISSION_EXECUTABLES}. Do "chmod {ALLOWED_PERMISSION_EXECUTABLES} {filename}" (or remove the shebang line).""" + ) + failed_tests += 1 + return failed_tests + + +def main() -> NoReturn: + failed_tests = 0 + failed_tests += check_all_filenames() + failed_tests += check_source_filenames() + failed_tests += check_all_file_permissions() + failed_tests += check_shebang_file_permissions() + + if failed_tests: + print( + f"ERROR: There were {failed_tests} failed tests in the lint-files.py lint test. Please resolve the above errors." + ) + sys.exit(1) + else: + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-files.sh b/test/lint/lint-files.sh new file mode 100755 index 0000000000..1e115778bd --- /dev/null +++ b/test/lint/lint-files.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +export LC_ALL=C + +set -e +cd "$(dirname $0)/../.." +test/lint/lint-files.py diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh index 5cfa41537f..c23b903bce 100755 --- a/test/lint/lint-include-guards.sh +++ b/test/lint/lint-include-guards.sh @@ -15,7 +15,7 @@ REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/ EXIT_CODE=0 for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") do - HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr "[:lower:]" "[:upper:]") + HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" if [[ $(grep -cE "^#(ifndef|define) ${HEADER_ID}" "${HEADER_FILE}") != 2 ]]; then echo "${HEADER_FILE} seems to be missing the expected include guard:" diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index e5657f7555..737d35a397 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -43,7 +43,7 @@ KNOWN_VIOLATIONS=( "src/dbwrapper.cpp.*stoul" "src/dbwrapper.cpp:.*vsnprintf" "src/httprpc.cpp.*trim" - "src/init.cpp:.*atoi" + "src/node/blockstorage.cpp:.*atoi" "src/qt/rpcconsole.cpp:.*atoi" "src/rest.cpp:.*strtol" "src/test/dbwrapper_tests.cpp:.*snprintf" diff --git a/test/lint/lint-shebang.sh b/test/lint/lint-shebang.sh deleted file mode 100755 index 13ebdfd78a..0000000000 --- a/test/lint/lint-shebang.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# Assert expected shebang lines - -export LC_ALL=C -EXIT_CODE=0 -for PYTHON_FILE in $(git ls-files -- "*.py"); do - if [[ $(head -c 2 "${PYTHON_FILE}") == "#!" && - $(head -n 1 "${PYTHON_FILE}") != "#!/usr/bin/env python3" ]]; then - echo "Missing shebang \"#!/usr/bin/env python3\" in ${PYTHON_FILE} (do not use python or python2)" - EXIT_CODE=1 - fi -done -for SHELL_FILE in $(git ls-files -- "*.sh"); do - if [[ $(head -n 1 "${SHELL_FILE}") != "#!/usr/bin/env bash" && - $(head -n 1 "${SHELL_FILE}") != "#!/bin/sh" ]]; then - echo "Missing expected shebang \"#!/usr/bin/env bash\" or \"#!/bin/sh\" in ${SHELL_FILE}" - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 27885b094e..b66ca00376 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -5,6 +5,8 @@ # names can be used. # See https://github.com/google/sanitizers/issues/1364 signed-integer-overflow:txmempool.cpp +# https://github.com/bitcoin/bitcoin/pull/21798#issuecomment-829180719 +signed-integer-overflow:policy/feerate.cpp # -fsanitize=integer suppressions # =============================== @@ -85,6 +87,7 @@ implicit-signed-integer-truncation:chain.h implicit-signed-integer-truncation:crypto/ implicit-signed-integer-truncation:cuckoocache.h implicit-signed-integer-truncation:leveldb/ +implicit-signed-integer-truncation:miner.cpp implicit-signed-integer-truncation:net.cpp implicit-signed-integer-truncation:net_processing.cpp implicit-signed-integer-truncation:streams.h @@ -96,6 +99,7 @@ implicit-unsigned-integer-truncation:crypto/ implicit-unsigned-integer-truncation:leveldb/ # std::variant warning fixed in https://github.com/gcc-mirror/gcc/commit/074436cf8cdd2a9ce75cadd36deb8301f00e55b9 implicit-unsigned-integer-truncation:std::__detail::__variant::_Variant_storage +shift-base:nanobench.h shift-base:*/include/c++/ shift-base:arith_uint256.cpp shift-base:crypto/ |