diff options
441 files changed, 8356 insertions, 2543 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 777eebd2c3..eeb2d6590b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,10 +16,8 @@ environment: VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' cache: - C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt -- C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in - C:\Qt5.9.8_x64_static_vs2019 install: -- cmd: pip install --quiet git+https://github.com/frerich/clcache.git@v4.2.0 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq # Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is: @@ -46,7 +44,6 @@ install: } c:\tools\vcpkg\vcpkg integrate install before_build: -- ps: clcache -M 536870912 # Powershell block below is to download and extract the Qt static libraries. The pseudo code is: # 1. If the Qt destination directory exists assume it is correct and do nothing. To # force a fresh install of the packages delete the job's appveyor cache. @@ -72,21 +69,20 @@ before_build: Write-Host "Qt binaries already present."; } - cmd: python build_msvc\msvc-autogen.py -- ps: Start-Process clcache-server -- ps: fsutil behavior set disablelastaccess 0 # Enable Access time feature on Windows (for clcache) build_script: -- cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\bitcoin.sln /m /v:q /nologo +- cmd: msbuild /p:TrackFileAccess=false build_msvc\bitcoin.sln /m /v:q /nologo after_build: -- ps: fsutil behavior set disablelastaccess 1 # Disable Access time feature on Windows (better performance) -- ps: clcache -z #- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe test_script: -- cmd: src\test_bitcoin.exe -k stdout -e stdout 2> NUL +- cmd: src\test_bitcoin.exe -l test_suite - cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL - ps: python test\util\bitcoin-util-test.py - cmd: python test\util\rpcauth-test.py # Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted. -- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation +# functional tests disabled for now. See +# https://github.com/bitcoin/bitcoin/pull/18626#issuecomment-613396202 +# https://github.com/bitcoin/bitcoin/issues/18623 +# - cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation artifacts: #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip deploy: off diff --git a/.cirrus.yml b/.cirrus.yml index f4a3878ed8..b839bfa5fb 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,7 +1,7 @@ task: - name: "FreeBsd 12.0 amd64 [GOAL: install] [no depends, only system libs]" + name: "FreeBsd 12.1 amd64 [GOAL: install] [no depends, only system libs]" freebsd_instance: - image: freebsd-12-0-release-amd64 + image_family: freebsd-12-1 # https://cirrus-ci.org/guide/FreeBSD/ cpu: 8 memory: 8G timeout_in: 60m @@ -28,6 +28,21 @@ task: - gmake check ${MAKEJOBS} VERBOSE=1 functional_test_script: - ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast +#task: +# name: "Windows" +# windows_container: +# image: cirrusci/windowsservercore:2019 +# env: +# CIRRUS_SHELL: powershell +# PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%' +# PYTHONUTF8: 1 +# QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' +# QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' +# QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' +# VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' +# VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' +# install_script: +# - choco install python --version=3.7.7 -y task: name: "x86_64 Linux [GOAL: install] [bionic] [Using ./ci/ system]" container: @@ -37,7 +52,7 @@ task: timeout_in: 60m env: MAKEJOBS: "-j9" - RUN_CI_ON_HOST: "1" + DANGER_RUN_CI_ON_HOST: "1" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" diff --git a/.travis.yml b/.travis.yml index 878f5ce518..ea589b4059 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,9 +38,6 @@ cache: - $TRAVIS_BUILD_DIR/depends/sdk-sources - $TRAVIS_BUILD_DIR/ci/scratch/.ccache - $TRAVIS_BUILD_DIR/releases/$HOST - # macOS - - $HOME/Library/Caches/Homebrew - - /usr/local/Homebrew before_cache: - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then brew cleanup; fi stages: @@ -74,7 +71,7 @@ jobs: - stage: lint name: 'lint' env: - cache: false + cache: pip language: python python: '3.5' # Oldest supported version according to doc/dependencies.md install: @@ -91,6 +88,26 @@ jobs: FILE_ENV="./ci/test/00_setup_env_arm.sh" QEMU_USER_CMD="" + - stage: test + name: 's390x native BE [GOAL: install] [bionic] [no depends, no GUI]' + arch: s390x + dist: bionic + addons: + apt: + packages: + - bsdmainutils + - libboost-filesystem-dev + - libboost-system-dev + - libboost-test-dev + - libboost-thread-dev + - libdb++-dev + - libdb-dev + - libevent-dev + env: >- + DANGER_RUN_CI_ON_HOST=true + CI_USE_APT_INSTALL=no + FILE_ENV="./ci/test/00_setup_env_s390x_host.sh" + # s390 build was disabled temporarily because of disk space issues on the Travis VM # # - stage: test @@ -158,6 +175,12 @@ jobs: # Xcode 11.3.1, macOS 10.14, SDK 10.15 # https://docs.travis-ci.com/user/reference/osx/#macos-version osx_image: xcode11.3 + cache: + directories: + - $TRAVIS_BUILD_DIR/ci/scratch/.ccache + - $TRAVIS_BUILD_DIR/releases/$HOST + - $HOME/Library/Caches/Homebrew + - /usr/local/Homebrew addons: homebrew: packages: @@ -171,4 +194,6 @@ jobs: - ccache - zeromq env: >- + DANGER_RUN_CI_ON_HOST=true + CI_USE_APT_INSTALL=no FILE_ENV="./ci/test/00_setup_env_mac_host.sh" diff --git a/build-aux/m4/ax_boost_base.m4 b/build-aux/m4/ax_boost_base.m4 index 16fa69b41f..2ae33f7140 100644 --- a/build-aux/m4/ax_boost_base.m4 +++ b/build-aux/m4/ax_boost_base.m4 @@ -33,7 +33,7 @@ # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 47 +#serial 48 # example boost program (need to pass version) m4_define([_AX_BOOST_BASE_PROGRAM], @@ -123,6 +123,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ dnl are almost assuredly the ones desired. AS_CASE([${host_cpu}], [i?86],[multiarch_libsubdir="lib/i386-${host_os}"], + [armv7l],[multiarch_libsubdir="lib/arm-${host_os}"], [multiarch_libsubdir="lib/${host_cpu}-${host_os}"] ) diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 5f0640ac27..c8b2fb1658 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ #define CLIENT_VERSION_MAJOR 0 /* Minor version */ -#define CLIENT_VERSION_MINOR 19 +#define CLIENT_VERSION_MINOR 20 /* Build revision */ #define CLIENT_VERSION_REVISION 99 diff --git a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in index adf4fa0354..6ec40461c2 100644 --- a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in +++ b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in @@ -8,6 +8,7 @@ <ConfigurationType>StaticLibrary</ConfigurationType> </PropertyGroup> <ItemGroup> + <ClCompile Include="..\..\src\util\url.cpp" /> @SOURCE_FILES@ </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/build_msvc/testconsensus/testconsensus.cpp b/build_msvc/testconsensus/testconsensus.cpp index 571d19957f..5fdd97dc78 100644 --- a/build_msvc/testconsensus/testconsensus.cpp +++ b/build_msvc/testconsensus/testconsensus.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 4c22e4e6c5..e0e555bbd1 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -24,7 +24,7 @@ echo "Fallback to default values in env (if not yet set)" export MAKEJOBS=${MAKEJOBS:--j4} # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) # This folder only exists on the ci host. -export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/} +export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch} # What host to compile for. See also ./depends/README.md # Tests that need cross-compilation export the appropriate HOST. # Tests that run natively guess the host @@ -53,7 +53,6 @@ export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} -export WINEDEBUG=${WINEDEBUG:-fixme-all} export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 2b30b4a5e3..b70a581532 100644 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 6ed6e40bd6..a4dc54d1c1 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index 5753c3af31..9ef4ba038f 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,9 +8,6 @@ export LC_ALL=C.UTF-8 export HOST=x86_64-apple-darwin16 export PIP_PACKAGES="zmq" -export RUN_CI_ON_HOST=true -export RUN_UNIT_TESTS=true -export RUN_FUNCTIONAL_TESTS=false export GOAL="install" export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" # Run without depends diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index d5f39daaf5..28c63f1cf6 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index c8c75dea1a..43ee219ef9 100644 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index eb77dc5b16..c27d525003 100644 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 6bb371920c..9c2be4cfac 100644 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 21c15236d2..87c8b70998 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,6 +10,8 @@ export CONTAINER_NAME=ci_native_qt5 export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash +export RUN_UNIT_TESTS_SEQUENTIAL="true" +export RUN_UNIT_TESTS="false" export GOAL="install" export TEST_PREVIOUS_RELEASES=true export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 4d3f345ca6..73ab5eebb6 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,11 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:16.04 -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang-8 llvm-8 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++" - -# xenial comes with old clang versions that can not parse the sanitizer suppressions files -# Remove unparseable lines as a hacky workaround -sed -i '/^implicit-/d' "${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan" +export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang-8 CXX=clang++-8" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 2a7b32cefc..d5b38a1d88 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,6 +10,10 @@ export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +if [[ "${TRAVIS}" == "true" && "${TRAVIS_REPO_SLUG}" != "bitcoin/bitcoin" ]]; then + export TEST_RUNNER_EXTRA="wallet_disable" # Only run wallet_disable as a smoke test to not hit the 50 min travis time limit +else + export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +fi export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 72ad99fe5d..c180d023de 100644 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_s390x_host.sh b/ci/test/00_setup_env_s390x_host.sh new file mode 100644 index 0000000000..3e31867f2f --- /dev/null +++ b/ci/test/00_setup_env_s390x_host.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=s390x-linux-gnu +export NO_DEPENDS=1 +export BITCOIN_CONFIG="--with-incompatible-bdb --enable-reduce-exports" +export RUN_UNIT_TESTS=true +export RUN_FUNCTIONAL_TESTS=true +export GOAL="install" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index a34933731c..8f0c62a1a6 100644 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index acf7eeb920..32eaaf15bb 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -26,7 +26,7 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order= export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" -env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|TEST_PREVIOUS_RELEASES|PREVIOUS_RELEASES_DIR)' | tee /tmp/env +env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|TEST_PREVIOUS_RELEASES|PREVIOUS_RELEASES_DIR)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN" elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) @@ -35,7 +35,7 @@ fi export P_CI_DIR="$PWD" -if [ -z "$RUN_CI_ON_HOST" ]; then +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" @@ -66,7 +66,7 @@ fi if [[ $DOCKER_NAME_TAG == centos* ]]; then ${CI_RETRY_EXE} DOCKER_EXEC yum -y install epel-release ${CI_RETRY_EXE} DOCKER_EXEC yum -y install $DOCKER_PACKAGES $PACKAGES -elif [ "$TRAVIS_OS_NAME" != "osx" ]; then +elif [ "$CI_USE_APT_INSTALL" != "no" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get update ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES fi @@ -77,6 +77,7 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then else DOCKER_EXEC free -m -h DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) + DOCKER_EXEC echo $(lscpu | grep Endian) DOCKER_EXEC echo "Free disk space:" DOCKER_EXEC df -h fi @@ -90,7 +91,7 @@ export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" -if [ -z "$RUN_CI_ON_HOST" ]; then +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 933f4cea91..17853e75dd 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index b6043cd61e..f568dbe37d 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -1,15 +1,13 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" -if [ -z "$NO_DEPENDS" ]; then - DOCKER_EXEC ccache --max-size=$CCACHE_SIZE -fi +DOCKER_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" BEGIN_FOLD autogen if [ -n "$CONFIG_SHELL" ]; then @@ -44,3 +42,9 @@ trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) END_FOLD + +BEGIN_FOLD cache_stats +DOCKER_EXEC "ccache --version | head -n 1 && ccache --show-stats" +DOCKER_EXEC du -sh "${DEPENDS_DIR}"/*/ +DOCKER_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" +END_FOLD diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 3b32513353..0f03d1770e 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,13 +21,20 @@ if [ -n "$USE_VALGRIND" ]; then END_FOLD fi +bash -c "${CI_WAIT}" & # Print dots in case the tests take a long time to run + if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests - bash -c "${CI_WAIT}" & # Print dots in case the unit tests take a long time to run DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 END_FOLD fi +if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then + BEGIN_FOLD unit-tests-seq + DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_ROOT_DIR}/build/bitcoin-*/src/test/test_bitcoin" --catch_system_errors=no -l test_suite + END_FOLD +fi + if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then BEGIN_FOLD functional-tests DOCKER_EXEC test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index f1d3088748..be7d7fcc1f 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/configure.ac b/configure.ac index a2d33ff5b6..546d00de1b 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ([2.69]) define(_CLIENT_VERSION_MAJOR, 0) -define(_CLIENT_VERSION_MINOR, 19) +define(_CLIENT_VERSION_MINOR, 20) define(_CLIENT_VERSION_REVISION, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) @@ -256,7 +256,7 @@ AC_ARG_ENABLE([gprof], [enable_gprof=$enableval], [enable_gprof=no]) -dnl Pass compiler & liner flags that make builds deterministic +dnl Pass compiler & linker flags that make builds deterministic AC_ARG_ENABLE([determinism], [AS_HELP_STRING([--enable-determinism], [Enable compilation flags that make builds deterministic (default is no)])], @@ -861,6 +861,9 @@ AC_LINK_IFELSE([AC_LANG_SOURCE([ ] ) +dnl thread_local is currently disabled when building with glibc back compat. +dnl Our minimum supported glibc is 2.17, however support for thread_local +dnl did not arrive in glibc until 2.18. if test "x$use_thread_local" = xyes || { test "x$use_thread_local" = xauto && test "x$use_glibc_compat" = xno; }; then TEMP_LDFLAGS="$LDFLAGS" LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" @@ -1265,9 +1268,9 @@ if test x$use_pkgconfig = xyes; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - PKG_CHECK_MODULES([EVENT], [libevent],, [AC_MSG_ERROR(libevent not found.)]) + PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR(libevent version 2.0.21 or greater not found.)]) if test x$TARGET_OS != xwindows; then - PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)]) + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR(libevent_pthreads version 2.0.21 or greater not found.)]) fi fi @@ -1285,7 +1288,7 @@ if test x$use_pkgconfig = xyes; then else if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) + AC_CHECK_HEADER([event2/event.h], [use_libevent=yes], AC_MSG_ERROR(libevent headers missing),) AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) if test x$TARGET_OS != xwindows; then AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) @@ -1523,6 +1526,7 @@ AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) +AM_CONDITIONAL([USE_LIBEVENT],[test x$use_libevent = xyes]) AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes]) diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index 6afa4351e7..bc5f09a3e2 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 92120eaff7..084914f11a 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/devtools/previous_release.sh b/contrib/devtools/previous_release.sh index 5ddfdb4e73..b2ecc274fb 100755 --- a/contrib/devtools/previous_release.sh +++ b/contrib/devtools/previous_release.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index c05c38d513..9444271bdc 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -158,6 +158,17 @@ def check_PE_HIGH_ENTROPY_VA(executable): reqbits = 0 return (bits & reqbits) == reqbits +def check_PE_RELOC_SECTION(executable) -> bool: + '''Check for a reloc section. This is required for functional ASLR.''' + p = subprocess.Popen([OBJDUMP_CMD, '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + for line in stdout.splitlines(): + if '.reloc' in line: + return True + return False + def check_PE_NX(executable): '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' (arch,bits) = get_PE_dll_characteristics(executable) @@ -206,6 +217,37 @@ def check_MACHO_NX(executable) -> bool: return False return True +def check_MACHO_LAZY_BINDINGS(executable) -> bool: + ''' + Check for no lazy bindings. + We don't use or check for MH_BINDATLOAD. See #18295. + ''' + p = subprocess.Popen([OTOOL_CMD, '-l', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + + for line in stdout.splitlines(): + tokens = line.split() + if 'lazy_bind_off' in tokens or 'lazy_bind_size' in tokens: + if tokens[1] != '0': + return False + return True + +def check_MACHO_Canary(executable) -> bool: + ''' + Check for use of stack canary + ''' + p = subprocess.Popen([OTOOL_CMD, '-Iv', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + ok = False + for line in stdout.splitlines(): + if '___stack_chk_fail' in line: + ok = True + return ok + CHECKS = { 'ELF': [ ('PIE', check_ELF_PIE), @@ -216,12 +258,15 @@ CHECKS = { 'PE': [ ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), - ('NX', check_PE_NX) + ('NX', check_PE_NX), + ('RELOC_SECTION', check_PE_RELOC_SECTION) ], 'MACHO': [ ('PIE', check_MACHO_PIE), ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('NX', check_MACHO_NX) + ('NX', check_MACHO_NX), + ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), + ('Canary', check_MACHO_Canary) ] } diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index e2a8154f16..ea70b27941 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -49,13 +49,15 @@ class TestSecurityChecks(unittest.TestCase): cc = 'x86_64-w64-mingw32-gcc' write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va']), - (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA NX')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va']), - (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--no-high-entropy-va']), - (1, executable+': failed HIGH_ENTROPY_VA')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed HIGH_ENTROPY_VA RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), (0, '')) def test_MACHO(self): @@ -64,13 +66,17 @@ class TestSecurityChecks(unittest.TestCase): cc = 'clang' write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace', '-Wl,-allow_stack_execute']), - (1, executable+': failed PIE NOUNDEFS NX')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace']), - (1, executable+': failed PIE NOUNDEFS')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), + (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), + (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), + (1, executable+': failed PIE LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all']), (0, '')) if __name__ == '__main__': diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh index f5cd05a2c3..95b1553215 100755 --- a/contrib/devtools/test_deterministic_coverage.sh +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py index 75034616f7..e005cb96da 100755 --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 440ac0ce6c..f421372e10 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-linux-0.20" +name: "bitcoin-core-linux-0.21" enable_cache: true distro: "ubuntu" suites: @@ -46,7 +46,7 @@ script: | FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS_BASE="-static-libstdc++" + HOST_LDFLAGS_BASE="-static-libstdc++ -Wl,-O2" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 85072c9740..82f8f194fc 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-osx-0.20" +name: "bitcoin-core-osx-0.21" enable_cache: true distro: "ubuntu" suites: diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 9d96465742..6bcd126662 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -8,6 +8,7 @@ architectures: packages: - "libssl-dev" - "autoconf" +- "automake" - "libtool" - "pkg-config" remotes: diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 779de45dfa..54ad68a2a3 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-win-0.20" +name: "bitcoin-core-win-0.21" enable_cache: true distro: "ubuntu" suites: diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index ba3036a89f..0a2c1302c8 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -13,6 +13,7 @@ D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 +82921A4B88FD454B7EB8CE3C796C4109063D4EAF Jon Atack 32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli 4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon C42AFF7C61B3E44A1454CD3557AF762DB3353322 Karl-Johan Alm (kallewoof) @@ -29,6 +30,7 @@ D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille (Location: Leuven, Belgiu 133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost +9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami 79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko 71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 8500379025..9f99b36f88 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -220,8 +220,6 @@ repository and will likely put one up soon. [guix/env-setup]: https://www.gnu.org/software/guix/manual/en/html_node/Build-Environment-Setup.html [guix/substitutes]: https://www.gnu.org/software/guix/manual/en/html_node/Substitutes.html [guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html -[guix/inferiors]: https://www.gnu.org/software/guix/manual/en/html_node/Inferiors.html -[guix/channels]: https://www.gnu.org/software/guix/manual/en/html_node/Channels.html [guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html [debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644 diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 2daa8aba5e..e20b2a048d 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -13,33 +13,106 @@ make -C "${PWD}/depends" -j"$MAX_JOBS" download ${V:+V=1} ${SOURCES_PATH:+SOURCE # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. time-machine() { guix time-machine --url=https://github.com/dongcarl/guix.git \ - --commit=b3a7c72c8b2425f8ddb0fc6e3b1caeed40f86dee \ + --commit=b066c25026f21fb57677aa34692a5034338e7ee3 \ -- "$@" } -# Deterministically build Bitcoin Core for HOSTs (overriable by environment) -for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu}; do +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + output, deploy, and distsrc-* directories before rebuilding + +Hint: To blow everything away, you may want to use: + + $ git clean -xdff --exclude='/depends/SDKs/*' + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory. Practically speaking, this means that all ignored +and untracked files and directories will be wiped, allowing you to start anew. +EOF +} + +# Deterministically build Bitcoin Core for HOSTs (overridable by environment) +# shellcheck disable=SC2153 +for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32}; do # Display proper warning when the user interrupts the build - trap 'echo "** INT received while building ${host}, you may want to clean up the relevant output and distsrc-* directories before rebuilding"' INT - - # Run the build script 'contrib/guix/libexec/build.sh' in the build - # container specified by 'contrib/guix/manifest.scm' - # shellcheck disable=SC2086 - time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # shellcheck disable=SC2086 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + MAX_JOBS="$MAX_JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + ) done diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index ee207a957c..550b1b8f40 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash export LC_ALL=C set -e -o pipefail +export TZ=UTC # Check that environment variables assumed to be set by the environment are set echo "Building for platform triple ${HOST:?not set} with reference timestamp ${SOURCE_DATE_EPOCH:?not set}..." @@ -36,23 +37,41 @@ store_path() { --expression='s|"[[:space:]]*$||' } -# Determine output paths to use in CROSS_* environment variables -CROSS_GLIBC="$(store_path glibc-cross-${HOST})" -CROSS_GLIBC_STATIC="$(store_path glibc-cross-${HOST} static)" -CROSS_KERNEL="$(store_path linux-libre-headers-cross-${HOST})" -CROSS_GCC="$(store_path gcc-cross-${HOST})" -CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... -CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - # Set environment variables to point Guix's cross-toolchain to the right # includes/libs for $HOST -# -# NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because -# the limits.h in it is missing a '#include_next <limits.h>' -# -export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" -export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" -export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" +case "$HOST" in + *mingw*) + # Determine output paths to use in CROSS_* environment variables + CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + NATIVE_GCC="$(store_path gcc-glibc-2.27-toolchain)" + export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" + export CPATH="${NATIVE_GCC}/include" + + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + ;; + *linux*) + CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" + CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" + CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + # NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because + # the limits.h in it is missing a '#include_next <limits.h>' + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + ;; + *) + exit 1 ;; +esac # Sanity check CROSS_*_PATH directories IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" @@ -74,16 +93,20 @@ export GUIX_LD_WRAPPER_DISABLE_RPATH=yes [ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env # Determine the correct value for -Wl,--dynamic-linker for the current $HOST -glibc_dynamic_linker=$( - case "$HOST" in - i686-linux-gnu) echo /lib/ld-linux.so.2 ;; - x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; - arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; - aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; - riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - *) exit 1 ;; - esac -) +case "$HOST" in + *linux*) + glibc_dynamic_linker=$( + case "$HOST" in + i686-linux-gnu) echo /lib/ld-linux.so.2 ;; + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + *) exit 1 ;; + esac + ) + ;; +esac # Environment variables for determinism export QT_RCC_TEST=1 @@ -136,11 +159,27 @@ DISTNAME="$(basename "$SOURCEDIST" '.tar.gz')" # Binary Tarball Building # ########################### -# Similar flags to Gitian -CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" -HOST_CFLAGS="-O2 -g -ffile-prefix-map=${PWD}=." -HOST_CXXFLAGS="-O2 -g -ffile-prefix-map=${PWD}=." -HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++" +# CONFIGFLAGS +CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" +case "$HOST" in + *linux*) CONFIGFLAGS+=" --enable-glibc-back-compat" ;; +esac + +# CFLAGS +HOST_CFLAGS="-O2 -g" +case "$HOST" in + *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; + *mingw*) HOST_CFLAGS+=" -fno-ident" ;; +esac + +# CXXFLAGS +HOST_CXXFLAGS="$HOST_CFLAGS" + +# LDFLAGS +case "$HOST" in + *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; + *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; +esac # Make $HOST-specific native binaries from depends available in $PATH export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" @@ -160,7 +199,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" ${CONFIGFLAGS} \ CFLAGS="${HOST_CFLAGS}" \ CXXFLAGS="${HOST_CXXFLAGS}" \ - LDFLAGS="${HOST_LDFLAGS}" + ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool @@ -169,9 +208,21 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" # Perform basic ELF security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} - # Check that executables only contain allowed gcc, glibc and libstdc++ - # version symbols for Linux distro back-compatibility. - make -C src --jobs=1 check-symbols ${V:+V=1} + + case "$HOST" in + *linux*|*mingw*) + # Check that executables only contain allowed gcc, glibc and libstdc++ + # version symbols for Linux distro back-compatibility. + make -C src --jobs=1 check-symbols ${V:+V=1} + ;; + esac + + # Make the os-specific installers + case "$HOST" in + *mingw*) + make deploy ${V:+V=1} + ;; + esac # Setup the directory where our Bitcoin Core build for HOST will be # installed. This directory will also later serve as the input for our @@ -180,9 +231,21 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" mkdir -p "${INSTALLPATH}" # Install built Bitcoin Core to $INSTALLPATH make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + + case "$HOST" in + *mingw*) + cp -f --target-directory="$OUTDIR" ./*-setup-unsigned.exe + ;; + esac ( cd installed + case "$HOST" in + *mingw*) + mv --target-directory="$DISTNAME"/lib/ "$DISTNAME"/bin/*.dll + ;; + esac + # Prune libtool and object archives find . -name "lib*.la" -delete find . -name "lib*.a" -delete @@ -196,19 +259,60 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" find "${DISTNAME}/lib" -type f -print0 } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg - cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + case "$HOST" in + *mingw*) + cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" + ;; + *linux*) + cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + ;; + esac # Finally, deterministically produce {non-,}debug binary tarballs ready # for release - find "${DISTNAME}" -not -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) - find "${DISTNAME}" -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + case "$HOST" in + *mingw*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -not -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) + ;; + *linux*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + ;; + esac ) ) + +case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir unsigned + cp --target-directory=unsigned/ "$OUTDIR"/bitcoin-*-setup-unsigned.exe + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + ) + ;; +esac diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index c25ac2977b..86c1a8d27f 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -10,17 +10,34 @@ (gnu packages file) (gnu packages gawk) (gnu packages gcc) + (gnu packages installers) (gnu packages linux) + (gnu packages mingw) (gnu packages perl) (gnu packages pkg-config) (gnu packages python) (gnu packages shells) + (guix build-system gnu) (guix build-system trivial) (guix gexp) (guix packages) (guix profiles) (guix utils)) +(define (make-ssp-fixed-gcc xgcc) + "Given a XGCC package, return a modified package that uses the SSP function +from glibc instead of from libssp.so. Our `symbol-check' script will complain if +we link against libssp.so, and thus will ensure that this works properly. + +Taken from: +http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" + (package + (inherit xgcc) + (arguments + (substitute-keyword-arguments (package-arguments xgcc) + ((#:make-flags flags) + `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) + (define (make-gcc-rpath-link xgcc) "Given a XGCC package, return a modified package that replace each instance of -rpath in the default system spec that's inserted by Guix with -rpath-link" @@ -102,45 +119,77 @@ desirable for building Bitcoin Core release binaries." base-libc base-gcc)) +(define (make-gcc-with-pthreads gcc) + (package-with-extra-configure-variable gcc "--enable-threads" "posix")) + +(define (make-mingw-pthreads-cross-toolchain target) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + (pthreads-xlibc mingw-w64-x86_64-winpthreads) + (pthreads-xgcc (make-gcc-with-pthreads + (cross-gcc target + #:xgcc (make-ssp-fixed-gcc gcc-9) + #:xbinutils xbinutils + #:libc pthreads-xlibc)))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-posix-toolchain")) + (version (package-version pthreads-xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + `(("binutils" ,xbinutils) + ("libc" ,pthreads-xlibc) + ("gcc" ,pthreads-xgcc))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page pthreads-xgcc)) + (license (package-license pthreads-xgcc))))) + + (packages->manifest - (list ;; The Basics - bash-minimal - which - coreutils - util-linux - ;; File(system) inspection - file - grep - diffutils - findutils - ;; File transformation - patch - gawk - sed - ;; Compression and archiving - tar - bzip2 - gzip - xz - zlib - ;; Build tools - gnu-make - libtool - autoconf - automake - pkg-config - ;; Scripting - perl - python-3.7 - ;; Native gcc 9 toolchain targeting glibc 2.27 - (make-gcc-toolchain gcc-9 glibc-2.27) - ;; Cross gcc 9 toolchains targeting glibc 2.27 - (make-bitcoin-cross-toolchain "i686-linux-gnu") - (make-bitcoin-cross-toolchain "x86_64-linux-gnu") - (make-bitcoin-cross-toolchain "aarch64-linux-gnu") - (make-bitcoin-cross-toolchain "arm-linux-gnueabihf") - ;; The glibc 2.27 for riscv64 needs gcc 7 to successfully build (see: - ;; https://www.gnu.org/software/gcc/gcc-7/changes.html#riscv). The final - ;; toolchain is still a gcc 9 toolchain targeting glibc 2.27. - (make-bitcoin-cross-toolchain "riscv64-linux-gnu" - #:base-gcc-for-libc gcc-7))) + (append + (list ;; The Basics + bash-minimal + which + coreutils + util-linux + ;; File(system) inspection + file + grep + diffutils + findutils + ;; File transformation + patch + gawk + sed + ;; Compression and archiving + tar + bzip2 + gzip + xz + zlib + ;; Build tools + gnu-make + libtool + autoconf + automake + pkg-config + ;; Scripting + perl + python-3.7 + ;; Native gcc 9 toolchain targeting glibc 2.27 + (make-gcc-toolchain gcc-9 glibc-2.27)) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + ;; Windows + (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64)) + ((string-contains target "riscv64-linux-") + (list (make-bitcoin-cross-toolchain "riscv64-linux-gnu" + #:base-gcc-for-libc gcc-7))) + ((string-contains target "-linux-") + (list (make-bitcoin-cross-toolchain target))) + (else '()))))) diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index bcca3b7cea..73f54cd488 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -2,7 +2,7 @@ # # linearize-data.py: Construct a linear, no-fork version of the chain. # -# Copyright (c) 2013-2019 The Bitcoin Core developers +# Copyright (c) 2013-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index f78bebf114..68ebb5def1 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -14,6 +14,10 @@ When complete, it will have produced `Bitcoin-Qt.dmg`. ## SDK Extraction +Our current macOS SDK (`macOSX10.14.sdk`) can be extracted from +[Xcode_10.2.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +An Apple ID is needed to download this. + `Xcode.app` is packaged in a `.xip` archive. This makes the SDK less-trivial to extract on non-macOS machines. One approach (tested on Debian Buster) is outlined below: @@ -38,14 +42,14 @@ xar -xf Xcode_10.2.1.xip -C . ./pbzx/pbzx -n Content | cpio -i -find Xcode.app -type d -name MacOSX.sdk -execdir sh -c 'tar -c MacOSX.sdk/ | gzip -9n > /MacOSX10.14.sdk.tar.gz' \; +find Xcode.app -type d -name MacOSX.sdk -exec sh -c 'tar --transform="s/MacOSX.sdk/MacOSX10.14.sdk/" -c -C$(dirname {}) MacOSX.sdk/ | gzip -9n > MacOSX10.14.sdk.tar.gz' \; ``` on macOS the process is more straightforward: ```bash xip -x Xcode_10.2.1.xip -tar -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk +tar -s "/MacOSX.sdk/MacOSX10.14.sdk/" -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk ``` Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh index 21243ada04..3c7bdf4217 100755 --- a/contrib/macdeploy/extract-osx-sdk.sh +++ b/contrib/macdeploy/extract-osx-sdk.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index c623d5b5e4..e8698994f1 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2013-2019 The Bitcoin Core developers +# Copyright (c) 2013-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh index a26791f0d1..78873dc0c3 100755 --- a/contrib/verify-commits/pre-push-hook.sh +++ b/contrib/verify-commits/pre-push-hook.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/doc/dependencies.md b/doc/dependencies.md index 51a2240107..0cb5311e8b 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -13,7 +13,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | | GCC | | [4.8+](https://gcc.gnu.org/) (C++11 support) | | | | | HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | +| libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | librsvg | | | | | | | MiniUPnPc | [2.0.20180203](https://miniupnp.tuxfamily.org/files) | | No | | | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index da07080724..b33b3ad18a 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -423,27 +423,52 @@ and its `cs_KeyStore` lock for example). Threads ------- -- ThreadScriptCheck : Verifies block scripts. +- [Main thread (`bitcoind`)](https://doxygen.bitcoincore.org/bitcoind_8cpp.html#a0ddf1224851353fc92bfbff6f499fa97) + : Started from `main()` in `bitcoind.cpp`. Responsible for starting up and + shutting down the application. -- ThreadImport : Loads blocks from `blk*.dat` files or `-loadblock=<file>`. +- [ThreadImport (`b-loadblk`)](https://doxygen.bitcoincore.org/init_8cpp.html#ae9e290a0e829ec0198518de2eda579d1) + : Loads blocks from `blk*.dat` files or `-loadblock=<file>` on startup. -- ThreadDNSAddressSeed : Loads addresses of peers from the DNS. +- [ThreadScriptCheck (`b-scriptch.x`)](https://doxygen.bitcoincore.org/validation_8cpp.html#a925a33e7952a157922b0bbb8dab29a20) + : Parallel script validation threads for transactions in blocks. -- ThreadMapPort : Universal plug-and-play startup/shutdown. +- [ThreadHTTP (`b-http`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#abb9f6ea8819672bd9a62d3695070709c) + : Libevent thread to listen for RPC and REST connections. -- ThreadSocketHandler : Sends/Receives data from peers on port 8333. +- [HTTP worker threads(`b-httpworker.x`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#aa6a7bc27265043bc0193220c5ae3a55f) + : Threads to service RPC and REST requests. -- ThreadOpenAddedConnections : Opens network connections to added nodes. +- [Indexer threads (`b-txindex`, etc)](https://doxygen.bitcoincore.org/class_base_index.html#a96a7407421fbf877509248bbe64f8d87) + : One thread per indexer. -- ThreadOpenConnections : Initiates new connections to peers. +- [SchedulerThread (`b-scheduler`)](https://doxygen.bitcoincore.org/class_c_scheduler.html#a14d2800815da93577858ea078aed1fba) + : Does asynchronous background tasks like dumping wallet contents, dumping + addrman and running asynchronous validationinterface callbacks. -- ThreadMessageHandler : Higher-level message handling (sending and receiving). +- [TorControlThread (`b-torcontrol`)](https://doxygen.bitcoincore.org/torcontrol_8cpp.html#a4faed3692d57a0d7bdbecf3b37f72de0) + : Libevent thread for tor connections. -- DumpAddresses : Dumps IP addresses of nodes to `peers.dat`. +- Net threads: -- ThreadRPCServer : Remote procedure call handler, listens on port 8332 for connections and services them. + - [ThreadMessageHandler (`b-msghand`)](https://doxygen.bitcoincore.org/class_c_connman.html#aacdbb7148575a31bb33bc345e2bf22a9) + : Application level message handling (sending and receiving). Almost + all net_processing and validation logic runs on this thread. -- Shutdown : Does an orderly shutdown of everything. + - [ThreadDNSAddressSeed (`b-dnsseed`)](https://doxygen.bitcoincore.org/class_c_connman.html#aa7c6970ed98a4a7bafbc071d24897d13) + : Loads addresses of peers from the DNS. + + - [ThreadMapPort (`b-upnp`)](https://doxygen.bitcoincore.org/net_8cpp.html#a63f82a71c4169290c2db1651a9bbe249) + : Universal plug-and-play startup/shutdown. + + - [ThreadSocketHandler (`b-net`)](https://doxygen.bitcoincore.org/class_c_connman.html#a765597cbfe99c083d8fa3d61bb464e34) + : Sends/Receives data from peers on port 8333. + + - [ThreadOpenAddedConnections (`b-addcon`)](https://doxygen.bitcoincore.org/class_c_connman.html#a0b787caf95e52a346a2b31a580d60a62) + : Opens network connections to added nodes. + + - [ThreadOpenConnections (`b-opencon`)](https://doxygen.bitcoincore.org/class_c_connman.html#a55e9feafc3bab78e5c9d408c207faa45) + : Initiates new connections to peers. Ignoring IDE/editor files -------------------------- @@ -965,7 +990,7 @@ Some good examples of scripted-diff: - [scripted-diff: Rename InitInterfaces to NodeContext](https://github.com/bitcoin/bitcoin/commit/301bd41a2e6765b185bd55f4c541f9e27aeea29d) uses an elegant script to replace occurrences of multiple terms in all source files. -- [scripted-diff: Remove g_connman, g_banman globals](https://github.com/bitcoin/bitcoin/commit/301bd41a2e6765b185bd55f4c541f9e27aeea29d) +- [scripted-diff: Remove g_connman, g_banman globals](https://github.com/bitcoin/bitcoin/commit/8922d7f6b751a3e6b3b9f6fb7961c442877fb65a) replaces specific terms in a list of specific source files. - [scripted-diff: Replace fprintf with tfm::format](https://github.com/bitcoin/bitcoin/commit/fac03ec43a15ad547161e37e53ea82482cc508f9) diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 9642337821..419b1db44e 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -133,3 +133,25 @@ $ afl/afl-fuzz -i inputs/ -o outputs/ -- src/test/fuzz/bech32 ``` Read the [`afl-fuzz` documentation](https://github.com/google/afl) for more information. + +# Fuzzing Bitcoin Core using Honggfuzz + +## Quickstart guide + +To quickly get started fuzzing Bitcoin Core using [Honggfuzz](https://github.com/google/honggfuzz): + +```sh +$ git clone https://github.com/bitcoin/bitcoin +$ cd bitcoin/ +$ ./autogen.sh +$ git clone https://github.com/google/honggfuzz +$ cd honggfuzz/ +$ make +$ cd .. +$ CC=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang CXX=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++ ./configure --enable-fuzz --with-sanitizers=address,undefined +$ make +$ mkdir -p inputs/ +$ honggfuzz/honggfuzz -i inputs/ -- src/test/fuzz/process_message +``` + +Read the [Honggfuzz documentation](https://github.com/google/honggfuzz/blob/master/docs/USAGE.md) for more information. diff --git a/doc/init.md b/doc/init.md index 0b327aab58..99aa0a0def 100644 --- a/doc/init.md +++ b/doc/init.md @@ -20,9 +20,9 @@ The macOS configuration assumes bitcoind will be set up for the current user. Configuration --------------------------------- -At a bare minimum, bitcoind requires that the rpcpassword setting be set -when running as a daemon. If the configuration file does not exist or this -setting is not set, bitcoind will shut down promptly after startup. +Running bitcoind as a daemon does not require any manual configuration. You may +set the `rpcauth` setting in the `bitcoin.conf` configuration file to override +the default behaviour of using a special cookie for authentication. This password does not have to be remembered or typed as it is mostly used as a fixed token that bitcoind and client programs read from the configuration diff --git a/doc/release-notes.md b/doc/release-notes.md index cd6a4d6b59..0d668a6302 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,5 +1,114 @@ -Please edit the release notes here: +*After branching off for a major version release of Bitcoin Core, use this +template to create the initial release notes draft.* -https://github.com/bitcoin-core/bitcoin-devwiki/wiki/0.20.0-Release-Notes-Draft +*The release notes draft is a temporary file that can be added to by anyone. See +[/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes) +for the process.* + +*Create the draft, named* "*version* Release Notes Draft" +*(e.g. "0.20.0 Release Notes Draft"), as a collaborative wiki in:* + +https://github.com/bitcoin-core/bitcoin-devwiki/wiki/ *Before the final release, move the notes back to this git repository.* + +*version* Release Notes Draft +=============================== + +Bitcoin Core version *version* is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-*version*/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes for older versions), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the datadir needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems using +the Linux kernel, macOS 10.12+, and Windows 7 and newer. It is not recommended +to use Bitcoin Core on unsupported systems. + +Bitcoin Core should also work on most other Unix-like systems but is not +as frequently tested on them. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + +In addition to previously supported CPU platforms, this release's pre-compiled +distribution provides binaries for the RISC-V platform. + +Notable changes +=============== + +P2P and network changes +----------------------- + +Updated RPCs +------------ + +Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. + +New RPCs +-------- + +Build System +------------ + +Updated settings +---------------- + +Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. + +New settings +------------ + +Wallet +------ + +#### Wallet RPC changes + +- The `upgradewallet` RPC replaces the `-upgradewallet` command line option. + (#15761) +- The `settxfee` RPC will fail if the fee was set higher than the `-maxtxfee` + command line setting. The wallet will already fail to create transactions + with fees higher than `-maxtxfee`. (#18467) + +GUI changes +----------- + +Low-level changes +================= + +Tests +----- + +Credits +======= + +Thanks to everyone who directly contributed to this release: + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/share/setup.nsi.in b/share/setup.nsi.in index dd9ee54d6d..4b2903a7c9 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -2,6 +2,11 @@ Name "@PACKAGE_NAME@ (64-bit)" RequestExecutionLevel highest SetCompressor /SOLID lzma +SetDateSave off + +# Uncomment these lines when investigating reproducibility errors +#SetCompress off +#SetDatablockOptimize off # General Symbol Definitions !define REGKEY "SOFTWARE\$(^Name)" diff --git a/src/Makefile.am b/src/Makefile.am index ae95902f67..627df97cad 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -215,6 +215,7 @@ BITCOIN_CORE_H = \ util/check.h \ util/error.h \ util/fees.h \ + util/golombrice.h \ util/spanparsing.h \ util/system.h \ util/macros.h \ @@ -523,9 +524,12 @@ libbitcoin_util_a_SOURCES = \ util/strencodings.cpp \ util/string.cpp \ util/time.cpp \ - util/url.cpp \ $(BITCOIN_CORE_H) +if USE_LIBEVENT +libbitcoin_util_a_SOURCES += util/url.cpp +endif + if GLIBC_BACK_COMPAT libbitcoin_util_a_SOURCES += compat/glibc_compat.cpp AM_LDFLAGS += $(COMPAT_LDFLAGS) diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index eae8b1fcd1..766c0fca54 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -12,6 +12,7 @@ GENERATED_BENCH_FILES = $(RAW_BENCH_FILES:.raw=.raw.h) bench_bench_bitcoin_SOURCES = \ $(RAW_BENCH_FILES) \ + bench/addrman.cpp \ bench/bench_bitcoin.cpp \ bench/bench.cpp \ bench/bench.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index c3021743f4..06dc59cf5c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -3,6 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. FUZZ_TARGETS = \ + test/fuzz/addition_overflow \ test/fuzz/addr_info_deserialize \ test/fuzz/addrdb \ test/fuzz/address_deserialize \ @@ -27,17 +28,22 @@ FUZZ_TARGETS = \ test/fuzz/bloom_filter \ test/fuzz/bloomfilter_deserialize \ test/fuzz/chain \ + test/fuzz/checkqueue \ test/fuzz/coins_deserialize \ + test/fuzz/cuckoocache \ test/fuzz/decode_tx \ test/fuzz/descriptor_parse \ test/fuzz/diskblockindex_deserialize \ test/fuzz/eval_script \ test/fuzz/fee_rate \ test/fuzz/fee_rate_deserialize \ + test/fuzz/fees \ test/fuzz/flat_file_pos_deserialize \ test/fuzz/flatfile \ test/fuzz/float \ + test/fuzz/golomb_rice \ test/fuzz/hex \ + test/fuzz/http_request \ test/fuzz/integer \ test/fuzz/inv_deserialize \ test/fuzz/key \ @@ -58,10 +64,13 @@ FUZZ_TARGETS = \ test/fuzz/parse_numbers \ test/fuzz/parse_script \ test/fuzz/parse_univalue \ + test/fuzz/prevector \ test/fuzz/partial_merkle_tree_deserialize \ test/fuzz/partially_signed_transaction_deserialize \ test/fuzz/pow \ test/fuzz/prefilled_transaction_deserialize \ + test/fuzz/primitives_transaction \ + test/fuzz/process_messages \ test/fuzz/process_message \ test/fuzz/process_message_addr \ test/fuzz/process_message_block \ @@ -194,6 +203,7 @@ BITCOIN_TESTS =\ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ + test/interfaces_tests.cpp \ test/key_io_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ @@ -238,7 +248,9 @@ BITCOIN_TESTS =\ test/uint256_tests.cpp \ test/util_tests.cpp \ test/validation_block_tests.cpp \ + test/validation_chainstatemanager_tests.cpp \ test/validation_flush_tests.cpp \ + test/validationinterface_tests.cpp \ test/versionbits_tests.cpp if ENABLE_WALLET @@ -279,6 +291,12 @@ endif if ENABLE_FUZZ +test_fuzz_addition_overflow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_addition_overflow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_addition_overflow_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_addition_overflow_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addition_overflow_SOURCES = test/fuzz/addition_overflow.cpp + test_fuzz_addr_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDR_INFO_DESERIALIZE=1 test_fuzz_addr_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addr_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -423,12 +441,24 @@ test_fuzz_chain_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_chain_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_chain_SOURCES = test/fuzz/chain.cpp +test_fuzz_checkqueue_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_checkqueue_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_checkqueue_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_checkqueue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_checkqueue_SOURCES = test/fuzz/checkqueue.cpp + test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_coins_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_cuckoocache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_cuckoocache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_cuckoocache_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_cuckoocache_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_cuckoocache_SOURCES = test/fuzz/cuckoocache.cpp + test_fuzz_decode_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_decode_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_decode_tx_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -465,6 +495,12 @@ test_fuzz_fee_rate_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_fee_rate_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_fee_rate_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_fees_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_fees_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_fees_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_fees_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_fees_SOURCES = test/fuzz/fees.cpp + test_fuzz_flat_file_pos_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFLAT_FILE_POS_DESERIALIZE=1 test_fuzz_flat_file_pos_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_flat_file_pos_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -483,12 +519,24 @@ test_fuzz_float_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_float_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_float_SOURCES = test/fuzz/float.cpp +test_fuzz_golomb_rice_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_golomb_rice_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_golomb_rice_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_golomb_rice_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_golomb_rice_SOURCES = test/fuzz/golomb_rice.cpp + test_fuzz_hex_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_hex_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_hex_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_hex_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_hex_SOURCES = test/fuzz/hex.cpp +test_fuzz_http_request_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_http_request_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_http_request_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_http_request_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_http_request_SOURCES = test/fuzz/http_request.cpp + test_fuzz_integer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_integer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_integer_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -609,6 +657,12 @@ test_fuzz_parse_univalue_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_parse_univalue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_parse_univalue_SOURCES = test/fuzz/parse_univalue.cpp +test_fuzz_prevector_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_prevector_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_prevector_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_prevector_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_prevector_SOURCES = test/fuzz/prevector.cpp + test_fuzz_partial_merkle_tree_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIAL_MERKLE_TREE_DESERIALIZE=1 test_fuzz_partial_merkle_tree_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_partial_merkle_tree_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -633,6 +687,18 @@ test_fuzz_prefilled_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_prefilled_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_primitives_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_primitives_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_primitives_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_primitives_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_primitives_transaction_SOURCES = test/fuzz/primitives_transaction.cpp + +test_fuzz_process_messages_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_process_messages_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_process_messages_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_process_messages_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_messages_SOURCES = test/fuzz/process_messages.cpp + test_fuzz_process_message_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_process_message_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 505d630b7d..d7bc73defb 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -11,6 +11,7 @@ TEST_UTIL_H = \ test/util/blockfilter.h \ test/util/logging.h \ test/util/mining.h \ + test/util/net.h \ test/util/setup_common.h \ test/util/str.h \ test/util/transaction_utils.h \ @@ -22,6 +23,7 @@ libtest_util_a_SOURCES = \ test/util/blockfilter.cpp \ test/util/logging.cpp \ test/util/mining.cpp \ + test/util/net.cpp \ test/util/setup_common.cpp \ test/util/str.cpp \ test/util/transaction_utils.cpp \ diff --git a/src/addrman.cpp b/src/addrman.cpp index 2f8a3a0bd5..02fb2fd491 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp new file mode 100644 index 0000000000..cc260df2b8 --- /dev/null +++ b/src/bench/addrman.cpp @@ -0,0 +1,140 @@ +// Copyright (c) 2020-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <addrman.h> +#include <bench/bench.h> +#include <random.h> +#include <util/time.h> + +#include <vector> + +/* A "source" is a source address from which we have received a bunch of other addresses. */ + +static constexpr size_t NUM_SOURCES = 64; +static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256; + +static std::vector<CAddress> g_sources; +static std::vector<std::vector<CAddress>> g_addresses; + +static void CreateAddresses() +{ + if (g_sources.size() > 0) { // already created + return; + } + + FastRandomContext rng(uint256(std::vector<unsigned char>(32, 123))); + + auto randAddr = [&rng]() { + in6_addr addr; + memcpy(&addr, rng.randbytes(sizeof(addr)).data(), sizeof(addr)); + + uint16_t port; + memcpy(&port, rng.randbytes(sizeof(port)).data(), sizeof(port)); + if (port == 0) { + port = 1; + } + + CAddress ret(CService(addr, port), NODE_NETWORK); + + ret.nTime = GetAdjustedTime(); + + return ret; + }; + + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + g_sources.emplace_back(randAddr()); + g_addresses.emplace_back(); + for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { + g_addresses[source_i].emplace_back(randAddr()); + } + } +} + +static void AddAddressesToAddrMan(CAddrMan& addrman) +{ + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + addrman.Add(g_addresses[source_i], g_sources[source_i]); + } +} + +static void FillAddrMan(CAddrMan& addrman) +{ + CreateAddresses(); + + AddAddressesToAddrMan(addrman); +} + +/* Benchmarks */ + +static void AddrManAdd(benchmark::State& state) +{ + CreateAddresses(); + + CAddrMan addrman; + + while (state.KeepRunning()) { + AddAddressesToAddrMan(addrman); + addrman.Clear(); + } +} + +static void AddrManSelect(benchmark::State& state) +{ + CAddrMan addrman; + + FillAddrMan(addrman); + + while (state.KeepRunning()) { + const auto& address = addrman.Select(); + assert(address.GetPort() > 0); + } +} + +static void AddrManGetAddr(benchmark::State& state) +{ + CAddrMan addrman; + + FillAddrMan(addrman); + + while (state.KeepRunning()) { + const auto& addresses = addrman.GetAddr(); + assert(addresses.size() > 0); + } +} + +static void AddrManGood(benchmark::State& state) +{ + /* Create many CAddrMan objects - one to be modified at each loop iteration. + * This is necessary because the CAddrMan::Good() method modifies the + * object, affecting the timing of subsequent calls to the same method and + * we want to do the same amount of work in every loop iteration. */ + + const uint64_t numLoops = state.m_num_iters * state.m_num_evals; + + std::vector<CAddrMan> addrmans(numLoops); + for (auto& addrman : addrmans) { + FillAddrMan(addrman); + } + + auto markSomeAsGood = [](CAddrMan& addrman) { + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { + if (addr_i % 32 == 0) { + addrman.Good(g_addresses[source_i][addr_i]); + } + } + } + }; + + uint64_t i = 0; + while (state.KeepRunning()) { + markSomeAsGood(addrmans.at(i)); + ++i; + } +} + +BENCHMARK(AddrManAdd, 5); +BENCHMARK(AddrManSelect, 1000000); +BENCHMARK(AddrManGetAddr, 500); +BENCHMARK(AddrManGood, 2); diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index f2fc3999fe..2107840a3a 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,8 +7,8 @@ #include <bech32.h> #include <util/strencodings.h> -#include <vector> #include <string> +#include <vector> static void Bech32Encode(benchmark::State& state) diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 5cf7e43f4b..7b93ef688d 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,6 @@ #include <numeric> #include <regex> -const RegTestingSetup* g_testing_setup = nullptr; const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; void benchmark::ConsolePrinter::header() @@ -115,18 +114,7 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double printer.header(); for (const auto& p : benchmarks()) { - RegTestingSetup test{}; - assert(g_testing_setup == nullptr); - g_testing_setup = &test; - { - LOCK(cs_main); - assert(::ChainActive().Height() == 0); - const bool witness_enabled{IsWitnessEnabled(::ChainActive().Tip(), Params().GetConsensus())}; - assert(witness_enabled); - } - if (!std::regex_match(p.first, baseMatch, reFilter)) { - g_testing_setup = nullptr; continue; } @@ -139,7 +127,6 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double p.second.func(state); } printer.result(state); - g_testing_setup = nullptr; } printer.footer(); diff --git a/src/bench/bench.h b/src/bench/bench.h index 6b7a0f76d1..629bca9a73 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,18 +5,15 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <chrono> #include <functional> #include <map> #include <string> #include <vector> -#include <chrono> #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> -struct RegTestingSetup; -extern const RegTestingSetup* g_testing_setup; //!< A pointer to the current testing setup - // Simple micro-benchmarking framework; API mostly matches a subset of the Google Benchmark // framework (see https://github.com/google/benchmark) // Why not use the Google Benchmark framework? Because adding Yet Another Dependency diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 9235d5fe6a..1b75854210 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,39 +17,40 @@ static const char* DEFAULT_PLOT_PLOTLYURL = "https://cdn.plot.ly/plotly-latest.m static const int64_t DEFAULT_PLOT_WIDTH = 1024; static const int64_t DEFAULT_PLOT_HEIGHT = 768; -static void SetupBenchArgs() +static void SetupBenchArgs(ArgsManager& argsman) { - SetupHelpOptions(gArgs); + SetupHelpOptions(argsman); - gArgs.AddArg("-list", "List benchmarks without executing them. Can be combined with -scaling and -filter", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-evals=<n>", strprintf("Number of measurement evaluations to perform. (default: %u)", DEFAULT_BENCH_EVALUATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-scaling=<n>", strprintf("Scaling factor for benchmark's runtime (default: %u)", DEFAULT_BENCH_SCALING), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-printer=(console|plot)", strprintf("Choose printer format. console: print data to console. plot: Print results as HTML graph (default: %s)", DEFAULT_BENCH_PRINTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-plotlyurl=<uri>", strprintf("URL to use for plotly.js (default: %s)", DEFAULT_PLOT_PLOTLYURL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-width=<x>", strprintf("Plot width in pixel (default: %u)", DEFAULT_PLOT_WIDTH), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-height=<x>", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-list", "List benchmarks without executing them. Can be combined with -scaling and -filter", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-evals=<n>", strprintf("Number of measurement evaluations to perform. (default: %u)", DEFAULT_BENCH_EVALUATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-scaling=<n>", strprintf("Scaling factor for benchmark's runtime (default: %u)", DEFAULT_BENCH_SCALING), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-printer=(console|plot)", strprintf("Choose printer format. console: print data to console. plot: Print results as HTML graph (default: %s)", DEFAULT_BENCH_PRINTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-plot-plotlyurl=<uri>", strprintf("URL to use for plotly.js (default: %s)", DEFAULT_PLOT_PLOTLYURL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-plot-width=<x>", strprintf("Plot width in pixel (default: %u)", DEFAULT_PLOT_WIDTH), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-plot-height=<x>", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } int main(int argc, char** argv) { - SetupBenchArgs(); + ArgsManager argsman; + SetupBenchArgs(argsman); std::string error; - if (!gArgs.ParseParameters(argc, argv, error)) { + if (!argsman.ParseParameters(argc, argv, error)) { tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); return EXIT_FAILURE; } - if (HelpRequested(gArgs)) { - std::cout << gArgs.GetHelpMessage(); + if (HelpRequested(argsman)) { + std::cout << argsman.GetHelpMessage(); return EXIT_SUCCESS; } - int64_t evaluations = gArgs.GetArg("-evals", DEFAULT_BENCH_EVALUATIONS); - std::string regex_filter = gArgs.GetArg("-filter", DEFAULT_BENCH_FILTER); - std::string scaling_str = gArgs.GetArg("-scaling", DEFAULT_BENCH_SCALING); - bool is_list_only = gArgs.GetBoolArg("-list", false); + int64_t evaluations = argsman.GetArg("-evals", DEFAULT_BENCH_EVALUATIONS); + std::string regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); + std::string scaling_str = argsman.GetArg("-scaling", DEFAULT_BENCH_SCALING); + bool is_list_only = argsman.GetBoolArg("-list", false); if (evaluations == 0) { return EXIT_SUCCESS; @@ -65,12 +66,12 @@ int main(int argc, char** argv) } std::unique_ptr<benchmark::Printer> printer = MakeUnique<benchmark::ConsolePrinter>(); - std::string printer_arg = gArgs.GetArg("-printer", DEFAULT_BENCH_PRINTER); + std::string printer_arg = argsman.GetArg("-printer", DEFAULT_BENCH_PRINTER); if ("plot" == printer_arg) { printer.reset(new benchmark::PlotlyPrinter( - gArgs.GetArg("-plot-plotlyurl", DEFAULT_PLOT_PLOTLYURL), - gArgs.GetArg("-plot-width", DEFAULT_PLOT_WIDTH), - gArgs.GetArg("-plot-height", DEFAULT_PLOT_HEIGHT))); + argsman.GetArg("-plot-plotlyurl", DEFAULT_PLOT_PLOTLYURL), + argsman.GetArg("-plot-width", DEFAULT_PLOT_WIDTH), + argsman.GetArg("-plot-height", DEFAULT_PLOT_HEIGHT))); } benchmark::BenchRunner::RunAll(*printer, evaluations, scaling_factor, regex_filter, is_list_only); diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index a113a73828..1a0084c915 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -16,6 +16,7 @@ static void AssembleBlock(benchmark::State& state) { + RegTestingSetup test_setup; const std::vector<unsigned char> op_true{OP_TRUE}; CScriptWitness witness; witness.stack.push_back(op_true); @@ -30,7 +31,7 @@ static void AssembleBlock(benchmark::State& state) std::array<CTransactionRef, NUM_BLOCKS - COINBASE_MATURITY + 1> txs; for (size_t b{0}; b < NUM_BLOCKS; ++b) { CMutableTransaction tx; - tx.vin.push_back(MineBlock(g_testing_setup->m_node, SCRIPT_PUB)); + tx.vin.push_back(MineBlock(test_setup.m_node, SCRIPT_PUB)); tx.vin.back().scriptWitness = witness; tx.vout.emplace_back(1337, SCRIPT_PUB); if (NUM_BLOCKS - b >= COINBASE_MATURITY) @@ -41,13 +42,13 @@ static void AssembleBlock(benchmark::State& state) for (const auto& txr : txs) { TxValidationState state; - bool ret{::AcceptToMemoryPool(::mempool, state, txr, nullptr /* plTxnReplaced */, false /* bypass_limits */, /* nAbsurdFee */ 0)}; + bool ret{::AcceptToMemoryPool(*test_setup.m_node.mempool, state, txr, nullptr /* plTxnReplaced */, false /* bypass_limits */, /* nAbsurdFee */ 0)}; assert(ret); } } while (state.KeepRunning()) { - PrepareBlock(g_testing_setup->m_node, SCRIPT_PUB); + PrepareBlock(test_setup.m_node, SCRIPT_PUB); } } diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index e9dd40293f..d658976c3c 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,6 +18,9 @@ // (https://github.com/bitcoin/bitcoin/issues/7883#issuecomment-224807484) static void CCoinsCaching(benchmark::State& state) { + const ECCVerifyHandle verify_handle; + ECC_Start(); + FillableSigningProvider keystore; CCoinsView coinsDummy; CCoinsViewCache coins(&coinsDummy); @@ -47,6 +50,7 @@ static void CCoinsCaching(benchmark::State& state) CAmount value = coins.GetValueIn(tx_1); assert(value == (50 + 21 + 22) * COIN); } + ECC_Stop(); } BENCHMARK(CCoinsCaching, 170 * 1000); diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp index a02a5315a4..df10f27d03 100644 --- a/src/bench/chacha_poly_aead.cpp +++ b/src/bench/chacha_poly_aead.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <crypto/poly1305.h> // for the POLY1305_TAGLEN constant #include <hash.h> -#include <limits> #include <assert.h> +#include <limits> /* Number of bytes to process per iteration */ static constexpr uint64_t BUFFER_SIZE_TINY = 64; diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 55786126b3..2b2c78905e 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,9 +6,9 @@ #include <bench/data.h> #include <chainparams.h> -#include <validation.h> -#include <streams.h> #include <consensus/validation.h> +#include <streams.h> +#include <validation.h> // These are the two major time-sinks which happen after we have fully received // a block off the wire, but before we can relay the block on to peers using diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index f5f96a0136..e052681181 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -1,14 +1,18 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <util/system.h> #include <checkqueue.h> +#include <key.h> #include <prevector.h> -#include <vector> -#include <boost/thread/thread.hpp> +#include <pubkey.h> #include <random.h> +#include <util/system.h> + +#include <boost/thread/thread.hpp> + +#include <vector> static const int MIN_CORES = 2; @@ -22,6 +26,9 @@ static const unsigned int QUEUE_BATCH_SIZE = 128; // and there is a little bit of work done between calls to Add. static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) { + const ECCVerifyHandle verify_handle; + ECC_Start(); + struct PrevectorJob { prevector<PREVECTOR_SIZE, uint8_t> p; PrevectorJob(){ @@ -57,5 +64,6 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) } tg.interrupt_all(); tg.join_all(); + ECC_Stop(); } BENCHMARK(CCheckQueueSpeedPrevectorJob, 1400); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index a9d4d78888..ddcef5121e 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <hash.h> -#include <random.h> -#include <uint256.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> #include <crypto/sha512.h> #include <crypto/siphash.h> +#include <hash.h> +#include <random.h> +#include <uint256.h> /* Number of bytes to hash per iteration */ static const uint64_t BUFFER_SIZE = 1000*1000; diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index a783370b4e..57673ccb84 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,13 +7,15 @@ #include <consensus/merkle.h> #include <consensus/validation.h> #include <pow.h> +#include <test/util/setup_common.h> #include <txmempool.h> #include <validation.h> - static void DuplicateInputs(benchmark::State& state) { + RegTestingSetup test_setup; + const CScript SCRIPT_PUB{CScript(OP_TRUE)}; const CChainParams& chainparams = Params(); diff --git a/src/bench/examples.cpp b/src/bench/examples.cpp index a2fdab5609..f88150200a 100644 --- a/src/bench/examples.cpp +++ b/src/bench/examples.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 1c9c106666..7df024def6 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <policy/policy.h> +#include <test/util/setup_common.h> #include <txmempool.h> @@ -24,6 +25,8 @@ static void AddTx(const CTransactionRef& tx, const CAmount& nFee, CTxMemPool& po // unique transactions for a more meaningful performance measurement. static void MempoolEviction(benchmark::State& state) { + RegTestingSetup test_setup; + CMutableTransaction tx1 = CMutableTransaction(); tx1.vin.resize(1); tx1.vin[0].scriptSig = CScript() << OP_1; diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 389e2c096f..38d8632318 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <policy/policy.h> +#include <test/util/setup_common.h> #include <txmempool.h> #include <vector> @@ -73,6 +74,7 @@ static void ComplexMemPool(benchmark::State& state) ordered_coins.emplace_back(MakeTransactionRef(tx)); available_coins.emplace_back(ordered_coins.back(), tx_counter++); } + TestingSetup test_setup; CTxMemPool pool; LOCK2(cs_main, pool.cs); while (state.KeepRunning()) { diff --git a/src/bench/merkle_root.cpp b/src/bench/merkle_root.cpp index bdb2bdbe3d..e84f92feae 100644 --- a/src/bench/merkle_root.cpp +++ b/src/bench/merkle_root.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <uint256.h> -#include <random.h> #include <consensus/merkle.h> +#include <random.h> +#include <uint256.h> static void MerkleRoot(benchmark::State& state) { diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index 2fc6f116a4..511573abac 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> #include <bench/data.h> -#include <validation.h> -#include <streams.h> #include <rpc/blockchain.h> +#include <streams.h> +#include <validation.h> #include <univalue.h> diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 0b34ae3f95..14bca5f7d1 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,6 +18,9 @@ // modified to measure performance of other types of scripts. static void VerifyScriptBench(benchmark::State& state) { + const ECCVerifyHandle verify_handle; + ECC_Start(); + const int flags = SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH; const int witnessversion = 0; @@ -69,6 +72,7 @@ static void VerifyScriptBench(benchmark::State& state) assert(csuccess == 1); #endif } + ECC_Stop(); } static void VerifyNestedIfScript(benchmark::State& state) { diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 0381369218..05d61fca22 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,7 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const bool add_watchonly, const bool add_mine) { + RegTestingSetup test_setup; const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; NodeContext node; @@ -30,8 +31,8 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const b if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); for (int i = 0; i < 100; ++i) { - generatetoaddress(g_testing_setup->m_node, address_mine.get_value_or(ADDRESS_WATCHONLY)); - generatetoaddress(g_testing_setup->m_node, ADDRESS_WATCHONLY); + generatetoaddress(test_setup.m_node, address_mine.get_value_or(ADDRESS_WATCHONLY)); + generatetoaddress(test_setup.m_node, ADDRESS_WATCHONLY); } SyncWithValidationInterfaceQueue(); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 6982eaab61..cdaabd6fab 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,6 +15,7 @@ #include <util/strencodings.h> #include <util/system.h> #include <util/translation.h> +#include <util/url.h> #include <functional> #include <memory> @@ -29,6 +30,7 @@ #include <compat/stdin.h> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = urlDecode; static const char DEFAULT_RPCCONNECT[] = "127.0.0.1"; static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; @@ -228,6 +230,7 @@ public: const int ID_NETWORKINFO = 0; const int ID_BLOCKCHAININFO = 1; const int ID_WALLETINFO = 2; + const int ID_BALANCES = 3; /** Create a simulated `getinfo` request. */ UniValue PrepareRequest(const std::string& method, const std::vector<std::string>& args) override @@ -239,6 +242,7 @@ public: result.push_back(JSONRPCRequestObj("getnetworkinfo", NullUniValue, ID_NETWORKINFO)); result.push_back(JSONRPCRequestObj("getblockchaininfo", NullUniValue, ID_BLOCKCHAININFO)); result.push_back(JSONRPCRequestObj("getwalletinfo", NullUniValue, ID_WALLETINFO)); + result.push_back(JSONRPCRequestObj("getbalances", NullUniValue, ID_BALANCES)); return result; } @@ -246,9 +250,9 @@ public: UniValue ProcessReply(const UniValue &batch_in) override { UniValue result(UniValue::VOBJ); - std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in, 3); - // Errors in getnetworkinfo() and getblockchaininfo() are fatal, pass them on - // getwalletinfo() is allowed to fail in case there is no wallet. + std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in, batch_in.size()); + // Errors in getnetworkinfo() and getblockchaininfo() are fatal, pass them on; + // getwalletinfo() and getbalances() are allowed to fail if there is no wallet. if (!batch[ID_NETWORKINFO]["error"].isNull()) { return batch[ID_NETWORKINFO]; } @@ -265,13 +269,15 @@ public: result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); if (!batch[ID_WALLETINFO]["result"].isNull()) { - result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); result.pushKV("keypoolsize", batch[ID_WALLETINFO]["result"]["keypoolsize"]); if (!batch[ID_WALLETINFO]["result"]["unlocked_until"].isNull()) { result.pushKV("unlocked_until", batch[ID_WALLETINFO]["result"]["unlocked_until"]); } result.pushKV("paytxfee", batch[ID_WALLETINFO]["result"]["paytxfee"]); } + if (!batch[ID_BALANCES]["result"].isNull()) { + result.pushKV("balance", batch[ID_BALANCES]["result"]["mine"]["trusted"]); + } result.pushKV("relayfee", batch[ID_NETWORKINFO]["result"]["relayfee"]); result.pushKV("warnings", batch[ID_NETWORKINFO]["result"]["warnings"]); return JSONRPCReplyObj(result, NullUniValue, 1); @@ -545,11 +551,19 @@ static int CommandLineRPC(int argc, char *argv[]) return nRet; } -int main(int argc, char* argv[]) -{ #ifdef WIN32 +// Export main() and ensure working ASLR on Windows. +// Exporting a symbol will prevent the linker from stripping +// the .reloc section from the binary, which is a requirement +// for ASLR. This is a temporary workaround until a fixed +// version of binutils is used for releases. +__declspec(dllexport) int main(int argc, char* argv[]) +{ util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); +#else +int main(int argc, char* argv[]) +{ #endif SetupEnvironment(); if (!SetupNetworking()) { diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index b4b2d7ed52..f54a299a36 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 7f1a4a114b..7f9439788a 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,11 +11,13 @@ #include <logging.h> #include <util/system.h> #include <util/translation.h> +#include <util/url.h> #include <wallet/wallettool.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = nullptr; static void SetupWalletToolArgs() { diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index e284dce0d5..2aa416ca44 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,10 +20,12 @@ #include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> +#include <util/url.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = urlDecode; static void WaitForShutdown(NodeContext& node) { @@ -51,7 +53,7 @@ static bool AppInit(int argc, char* argv[]) // Parameters // // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() - SetupServerArgs(); + SetupServerArgs(node); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { return InitError(strprintf("Error parsing command line arguments: %s\n", error)); diff --git a/src/blockencodings.h b/src/blockencodings.h index be50166cfc..377ac3a1a6 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 7aff3be6e7..5f5bed5bda 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -12,6 +12,7 @@ #include <primitives/transaction.h> #include <script/script.h> #include <streams.h> +#include <util/golombrice.h> /// SerType used to serialize parameters in GCS filter encoding. static constexpr int GCS_SER_TYPE = SER_NETWORK; @@ -23,37 +24,6 @@ static const std::map<BlockFilterType, std::string> g_filter_types = { {BlockFilterType::BASIC, "basic"}, }; -template <typename OStream> -static void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x) -{ - // Write quotient as unary-encoded: q 1's followed by one 0. - uint64_t q = x >> P; - while (q > 0) { - int nbits = q <= 64 ? static_cast<int>(q) : 64; - bitwriter.Write(~0ULL, nbits); - q -= nbits; - } - bitwriter.Write(0, 1); - - // Write the remainder in P bits. Since the remainder is just the bottom - // P bits of x, there is no need to mask first. - bitwriter.Write(x, P); -} - -template <typename IStream> -static uint64_t GolombRiceDecode(BitStreamReader<IStream>& bitreader, uint8_t P) -{ - // Read unary-encoded quotient: q 1's followed by one 0. - uint64_t q = 0; - while (bitreader.Read(1) == 1) { - ++q; - } - - uint64_t r = bitreader.Read(P); - - return (q << P) + r; -} - // Map a value x that is uniformly distributed in the range [0, 2^64) to a // value uniformly distributed in [0, n) by returning the upper 64 bits of // x * n. diff --git a/src/bloom.cpp b/src/bloom.cpp index bd6069b31f..30af507243 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -102,19 +102,6 @@ bool CBloomFilter::contains(const uint256& hash) const return contains(data); } -void CBloomFilter::clear() -{ - vData.assign(vData.size(),0); - isFull = false; - isEmpty = true; -} - -void CBloomFilter::reset(const unsigned int nNewTweak) -{ - clear(); - nTweak = nNewTweak; -} - bool CBloomFilter::IsWithinSizeConstraints() const { return vData.size() <= MAX_BLOOM_FILTER_SIZE && nHashFuncs <= MAX_HASH_FUNCS; diff --git a/src/bloom.h b/src/bloom.h index 68e76a0258..8e3b7be54d 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -84,9 +84,6 @@ public: bool contains(const COutPoint& outpoint) const; bool contains(const uint256& hash) const; - void clear(); - void reset(const unsigned int nNewTweak); - //! True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS //! (catch a filter which was just deserialized which was too big) bool IsWithinSizeConstraints() const; diff --git a/src/chain.h b/src/chain.h index 64c016a1d6..802e23f775 100644 --- a/src/chain.h +++ b/src/chain.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparams.cpp b/src/chainparams.cpp index f8f2d41138..799474fae2 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparams.h b/src/chainparams.h index 379c75e4be..542ef329da 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/checkqueue.h b/src/checkqueue.h index 9dab5a09ac..e3faa1dec0 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/coins.cpp b/src/coins.cpp index b71362c6a0..6b4cb2aec7 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -77,8 +77,21 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi } if (!possible_overwrite) { if (!it->second.coin.IsSpent()) { - throw std::logic_error("Adding new coin that replaces non-pruned entry"); + throw std::logic_error("Attempted to overwrite an unspent coin (when possible_overwrite is false)"); } + // If the coin exists in this cache as a spent coin and is DIRTY, then + // its spentness hasn't been flushed to the parent cache. We're + // re-adding the coin to this cache now but we can't mark it as FRESH. + // If we mark it FRESH and then spend it before the cache is flushed + // we would remove it from this cache and would never flush spentness + // to the parent cache. + // + // Re-adding a spent coin can happen in the case of a re-org (the coin + // is 'spent' when the block adding it is disconnected and then + // re-added when it is also added in a newly connected block). + // + // If the coin doesn't exist in the current cache, or is spent but not + // DIRTY, then it can be marked FRESH. fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); } it->second.coin = std::move(coin); @@ -86,12 +99,12 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); } -void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check) { +void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) { bool fCoinbase = tx.IsCoinBase(); const uint256& txid = tx.GetHash(); for (size_t i = 0; i < tx.vout.size(); ++i) { - bool overwrite = check ? cache.HaveCoin(COutPoint(txid, i)) : fCoinbase; - // Always set the possible_overwrite flag to AddCoin for coinbase txn, in order to correctly + bool overwrite = check_for_overwrite ? cache.HaveCoin(COutPoint(txid, i)) : fCoinbase; + // Coinbase transactions can always be overwritten, in order to correctly // deal with the pre-BIP30 occurrences of duplicate coinbase transactions. cache.AddCoin(COutPoint(txid, i), Coin(tx.vout[i], nHeight, fCoinbase), overwrite); } @@ -152,11 +165,11 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn } CCoinsMap::iterator itUs = cacheCoins.find(it->first); if (itUs == cacheCoins.end()) { - // The parent cache does not have an entry, while the child does - // We can ignore it if it's both FRESH and pruned in the child + // The parent cache does not have an entry, while the child cache does. + // We can ignore it if it's both spent and FRESH in the child if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { - // Otherwise we will need to create it in the parent - // and move the data up and mark it as dirty + // Create the coin in the parent cache, move the data up + // and mark it as dirty. CCoinsCacheEntry& entry = cacheCoins[it->first]; entry.coin = std::move(it->second.coin); cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); @@ -169,19 +182,18 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn } } } else { - // Assert that the child cache entry was not marked FRESH if the - // parent cache entry has unspent outputs. If this ever happens, - // it means the FRESH flag was misapplied and there is a logic - // error in the calling code. + // Found the entry in the parent cache if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) { - throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); + // The coin was marked FRESH in the child cache, but the coin + // exists in the parent cache. If this ever happens, it means + // the FRESH flag was misapplied and there is a logic error in + // the calling code. + throw std::logic_error("FRESH flag misapplied to coin that exists in parent cache"); } - // Found the entry in the parent cache if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { - // The grandparent does not have an entry, and the child is - // modified and being pruned. This means we can just delete - // it from the parent. + // The grandparent cache does not have an entry, and the coin + // has been spent. We can just delete it from the parent cache. cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { @@ -190,11 +202,10 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn itUs->second.coin = std::move(it->second.coin); cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; - // NOTE: It is possible the child has a FRESH flag here in - // the event the entry we found in the parent is pruned. But - // we must not copy that FRESH flag to the parent as that - // pruned state likely still needs to be communicated to the - // grandparent. + // NOTE: It isn't safe to mark the coin as FRESH in the parent + // cache. If it already existed and was spent in the parent + // cache then marking it FRESH would prevent that spentness + // from being flushed to the grandparent. } } } diff --git a/src/coins.h b/src/coins.h index ea2e759b54..2aed56c2bd 100644 --- a/src/coins.h +++ b/src/coins.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -109,19 +109,45 @@ public: } }; +/** + * A Coin in one level of the coins database caching hierarchy. + * + * A coin can either be: + * - unspent or spent (in which case the Coin object will be nulled out - see Coin.Clear()) + * - DIRTY or not DIRTY + * - FRESH or not FRESH + * + * Out of these 2^3 = 8 states, only some combinations are valid: + * - unspent, FRESH, DIRTY (e.g. a new coin created in the cache) + * - unspent, not FRESH, DIRTY (e.g. a coin changed in the cache during a reorg) + * - unspent, not FRESH, not DIRTY (e.g. an unspent coin fetched from the parent cache) + * - spent, FRESH, not DIRTY (e.g. a spent coin fetched from the parent cache) + * - spent, not FRESH, DIRTY (e.g. a coin is spent and spentness needs to be flushed to the parent) + */ struct CCoinsCacheEntry { Coin coin; // The actual cached data. unsigned char flags; enum Flags { - DIRTY = (1 << 0), // This cache entry is potentially different from the version in the parent view. - FRESH = (1 << 1), // The parent view does not have this entry (or it is pruned). - /* Note that FRESH is a performance optimization with which we can - * erase coins that are fully spent if we know we do not need to - * flush the changes to the parent cache. It is always safe to - * not mark FRESH if that condition is not guaranteed. + /** + * DIRTY means the CCoinsCacheEntry is potentially different from the + * version in the parent cache. Failure to mark a coin as DIRTY when + * it is potentially different from the parent cache will cause a + * consensus failure, since the coin's state won't get written to the + * parent when the cache is flushed. + */ + DIRTY = (1 << 0), + /** + * FRESH means the parent cache does not have this coin or that it is a + * spent coin in the parent cache. If a FRESH coin in the cache is + * later spent, it can be deleted entirely and doesn't ever need to be + * flushed to the parent. This is a performance optimization. Marking a + * coin as FRESH when it exists unspent in the parent cache will cause a + * consensus failure, since it might not be deleted from the parent + * when this cache is flushed. */ + FRESH = (1 << 1), }; CCoinsCacheEntry() : flags(0) {} @@ -246,7 +272,7 @@ public: bool HaveCoinInCache(const COutPoint &outpoint) const; /** - * Return a reference to Coin in the cache, or a pruned one if not found. This is + * Return a reference to Coin in the cache, or coinEmpty if not found. This is * more efficient than GetCoin. * * Generally, do not hold the reference returned for more than a short scope. @@ -258,10 +284,10 @@ public: const Coin& AccessCoin(const COutPoint &output) const; /** - * Add a coin. Set potential_overwrite to true if a non-pruned version may - * already exist. + * Add a coin. Set possible_overwrite to true if an unspent version may + * already exist in the cache. */ - void AddCoin(const COutPoint& outpoint, Coin&& coin, bool potential_overwrite); + void AddCoin(const COutPoint& outpoint, Coin&& coin, bool possible_overwrite); /** * Spend a coin. Pass moveto in order to get the deleted data. diff --git a/src/compressor.h b/src/compressor.h index 223603e7e9..478bfff0b6 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index 241cc316a6..af01902c92 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/core_read.cpp b/src/core_read.cpp index d036955641..df78c319ee 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/sha256_shani.cpp b/src/crypto/sha256_shani.cpp index 92f67710fb..3473f6e39f 100644 --- a/src/crypto/sha256_shani.cpp +++ b/src/crypto/sha256_shani.cpp @@ -11,13 +11,11 @@ #include <stdint.h> #include <immintrin.h> - - namespace { -const __m128i MASK = _mm_set_epi64x(0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL); -const __m128i INIT0 = _mm_set_epi64x(0x6a09e667bb67ae85ull, 0x510e527f9b05688cull); -const __m128i INIT1 = _mm_set_epi64x(0x3c6ef372a54ff53aull, 0x1f83d9ab5be0cd19ull); +alignas(__m128i) const uint8_t MASK[16] = {0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04, 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c}; +alignas(__m128i) const uint8_t INIT0[16] = {0x8c, 0x68, 0x05, 0x9b, 0x7f, 0x52, 0x0e, 0x51, 0x85, 0xae, 0x67, 0xbb, 0x67, 0xe6, 0x09, 0x6a}; +alignas(__m128i) const uint8_t INIT1[16] = {0x19, 0xcd, 0xe0, 0x5b, 0xab, 0xd9, 0x83, 0x1f, 0x3a, 0xf5, 0x4f, 0xa5, 0x72, 0xf3, 0x6e, 0x3c}; void inline __attribute__((always_inline)) QuadRound(__m128i& state0, __m128i& state1, uint64_t k1, uint64_t k0) { @@ -67,12 +65,12 @@ void inline __attribute__((always_inline)) Unshuffle(__m128i& s0, __m128i& s1) __m128i inline __attribute__((always_inline)) Load(const unsigned char* in) { - return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)in), MASK); + return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)in), _mm_load_si128((const __m128i*)MASK)); } void inline __attribute__((always_inline)) Save(unsigned char* out, __m128i s) { - _mm_storeu_si128((__m128i*)out, _mm_shuffle_epi8(s, MASK)); + _mm_storeu_si128((__m128i*)out, _mm_shuffle_epi8(s, _mm_load_si128((const __m128i*)MASK))); } } @@ -149,8 +147,8 @@ void Transform_2way(unsigned char* out, const unsigned char* in) __m128i bm0, bm1, bm2, bm3, bs0, bs1, bso0, bso1; /* Transform 1 */ - bs0 = as0 = INIT0; - bs1 = as1 = INIT1; + bs0 = as0 = _mm_load_si128((const __m128i*)INIT0); + bs1 = as1 = _mm_load_si128((const __m128i*)INIT1); am0 = Load(in); bm0 = Load(in + 64); QuadRound(as0, as1, am0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull); @@ -219,10 +217,10 @@ void Transform_2way(unsigned char* out, const unsigned char* in) ShiftMessageC(bm1, bm2, bm3); QuadRound(as0, as1, am3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); QuadRound(bs0, bs1, bm3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); - as0 = _mm_add_epi32(as0, INIT0); - bs0 = _mm_add_epi32(bs0, INIT0); - as1 = _mm_add_epi32(as1, INIT1); - bs1 = _mm_add_epi32(bs1, INIT1); + as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0)); + bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0)); + as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1)); + bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1)); /* Transform 2 */ aso0 = as0; @@ -275,8 +273,8 @@ void Transform_2way(unsigned char* out, const unsigned char* in) bm1 = bs1; /* Transform 3 */ - bs0 = as0 = INIT0; - bs1 = as1 = INIT1; + bs0 = as0 = _mm_load_si128((const __m128i*)INIT0); + bs1 = as1 = _mm_load_si128((const __m128i*)INIT1); QuadRound(as0, as1, am0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull); QuadRound(bs0, bs1, bm0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull); QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); @@ -339,10 +337,10 @@ void Transform_2way(unsigned char* out, const unsigned char* in) ShiftMessageC(bm1, bm2, bm3); QuadRound(as0, as1, am3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull); QuadRound(bs0, bs1, bm3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull); - as0 = _mm_add_epi32(as0, INIT0); - bs0 = _mm_add_epi32(bs0, INIT0); - as1 = _mm_add_epi32(as1, INIT1); - bs1 = _mm_add_epi32(bs1, INIT1); + as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0)); + bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0)); + as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1)); + bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1)); /* Extract hash into out */ Unshuffle(as0, as1); diff --git a/src/cuckoocache.h b/src/cuckoocache.h index 4ad5818cdc..2daf676c4a 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_CUCKOOCACHE_H #define BITCOIN_CUCKOOCACHE_H +#include <algorithm> // std::find #include <array> #include <atomic> #include <cmath> diff --git a/src/flatfile.h b/src/flatfile.h index d80682d383..60b3503cc3 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 4d49736140..60c4d06f12 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 11d73b7c9a..176284d103 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -189,7 +189,7 @@ static bool InitHTTPAllowList() } /** HTTP request method as string - use for logging only */ -static std::string RequestMethodString(HTTPRequest::RequestMethod m) +std::string RequestMethodString(HTTPRequest::RequestMethod m) { switch (m) { case HTTPRequest::GET: @@ -510,10 +510,10 @@ void HTTPEvent::trigger(struct timeval* tv) else evtimer_add(ev, tv); // trigger after timeval passed } -HTTPRequest::HTTPRequest(struct evhttp_request* _req) : req(_req), - replySent(false) +HTTPRequest::HTTPRequest(struct evhttp_request* _req, bool _replySent) : req(_req), replySent(_replySent) { } + HTTPRequest::~HTTPRequest() { if (!replySent) { diff --git a/src/httpserver.h b/src/httpserver.h index 46820e6aee..97cd63778a 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -60,7 +60,7 @@ private: bool replySent; public: - explicit HTTPRequest(struct evhttp_request* req); + explicit HTTPRequest(struct evhttp_request* req, bool replySent = false); ~HTTPRequest(); enum RequestMethod { diff --git a/src/index/base.cpp b/src/index/base.cpp index ba71830b6e..7bff463f5b 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/index/base.h b/src/index/base.h index 95d83b9b47..3fab810bb2 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/indirectmap.h b/src/indirectmap.h index 417d500bd4..a9f4b0b9f2 100644 --- a/src/indirectmap.h +++ b/src/indirectmap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/init.cpp b/src/init.cpp index 437e934093..a213dacbe0 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -243,13 +243,12 @@ void Shutdown(NodeContext& node) } // FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing - // - // g_chainstate is referenced here directly (instead of ::ChainstateActive()) because it - // may not have been initialized yet. { LOCK(cs_main); - if (g_chainstate && g_chainstate->CanFlushToDisk()) { - g_chainstate->ForceFlushStateToDisk(); + for (CChainState* chainstate : g_chainman.GetAll()) { + if (chainstate->CanFlushToDisk()) { + chainstate->ForceFlushStateToDisk(); + } } } @@ -273,9 +272,11 @@ void Shutdown(NodeContext& node) { LOCK(cs_main); - if (g_chainstate && g_chainstate->CanFlushToDisk()) { - g_chainstate->ForceFlushStateToDisk(); - g_chainstate->ResetCoinsViews(); + for (CChainState* chainstate : g_chainman.GetAll()) { + if (chainstate->CanFlushToDisk()) { + chainstate->ForceFlushStateToDisk(); + chainstate->ResetCoinsViews(); + } } pblocktree.reset(); } @@ -291,20 +292,23 @@ void Shutdown(NodeContext& node) } #endif - try { - if (!fs::remove(GetPidFile())) { - LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); - } - } catch (const fs::filesystem_error& e) { - LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); - } node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); globalVerifyHandle.reset(); ECC_Stop(); + node.args = nullptr; if (node.mempool) node.mempool = nullptr; node.scheduler.reset(); + + try { + if (!fs::remove(GetPidFile())) { + LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); + } + } catch (const fs::filesystem_error& e) { + LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); + } + LogPrintf("%s: done\n", __func__); } @@ -357,8 +361,11 @@ static void OnRPCStopped() LogPrint(BCLog::RPC, "RPC stopped.\n"); } -void SetupServerArgs() +void SetupServerArgs(NodeContext& node) { + assert(!node.args); + node.args = &gArgs; + SetupHelpOptions(gArgs); gArgs.AddArg("-help-debug", "Print help message with debugging options and exit", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); // server-only for now @@ -513,11 +520,16 @@ void SetupServerArgs() gArgs.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " - "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + ListLogCategories() + ".", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ".", + ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +#ifdef HAVE_THREAD_LOCAL gArgs.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (only available on platforms supporting thread_local) (default: %u)", DEFAULT_LOGTHREADNAMES), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +#else + hidden_args.emplace_back("-logthreadnames"); +#endif gArgs.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -717,11 +729,17 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) } // scan for better chains in the block chain database, that are not yet connected in the active best chain - BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { - LogPrintf("Failed to connect best block (%s)\n", state.ToString()); - StartShutdown(); - return; + + // We can't hold cs_main during ActivateBestChain even though we're accessing + // the g_chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve + // the relevant pointers before the ABC call. + for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + BlockValidationState state; + if (!chainstate->ActivateBestChain(state, chainparams, nullptr)) { + LogPrintf("Failed to connect best block (%s)\n", state.ToString()); + StartShutdown(); + return; + } } if (gArgs.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { @@ -849,7 +867,9 @@ void InitLogging() LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", !gArgs.GetBoolArg("-daemon", false)); LogInstance().m_log_timestamps = gArgs.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); LogInstance().m_log_time_micros = gArgs.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); +#ifdef HAVE_THREAD_LOCAL LogInstance().m_log_threadnames = gArgs.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); +#endif fLogIPs = gArgs.GetBoolArg("-logips", DEFAULT_LOGIPS); @@ -1511,17 +1531,18 @@ bool AppInitMain(NodeContext& node) bool fLoaded = false; while (!fLoaded && !ShutdownRequested()) { bool fReset = fReindex; + auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); + }; std::string strLoadError; uiInterface.InitMessage(_("Loading block index...").translated); do { const int64_t load_block_index_start_time = GetTimeMillis(); - bool is_coinsview_empty; try { LOCK(cs_main); - // This statement makes ::ChainstateActive() usable. - g_chainstate = MakeUnique<CChainState>(); + g_chainman.InitializeChainstate(); UnloadBlockIndex(); // new CBlockTreeDB tries to delete the existing file, which @@ -1574,43 +1595,53 @@ bool AppInitMain(NodeContext& node) // At this point we're either in reindex or we've loaded a useful // block tree into BlockIndex()! - ::ChainstateActive().InitCoinsDB( - /* cache_size_bytes */ nCoinDBCache, - /* in_memory */ false, - /* should_wipe */ fReset || fReindexChainState); - - ::ChainstateActive().CoinsErrorCatcher().AddReadErrCallback([]() { - uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down.").translated, - "", CClientUIInterface::MSG_ERROR); - }); - - // If necessary, upgrade from older database format. - // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!::ChainstateActive().CoinsDB().Upgrade()) { - strLoadError = _("Error upgrading chainstate database").translated; - break; - } - - // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!::ChainstateActive().ReplayBlocks(chainparams)) { - strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; - break; - } - - // The on-disk coinsdb is now in a good state, create the cache - ::ChainstateActive().InitCoinsCache(); - assert(::ChainstateActive().CanFlushToDisk()); + bool failed_chainstate_init = false; + + for (CChainState* chainstate : g_chainman.GetAll()) { + LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); + chainstate->InitCoinsDB( + /* cache_size_bytes */ nCoinDBCache, + /* in_memory */ false, + /* should_wipe */ fReset || fReindexChainState); + + chainstate->CoinsErrorCatcher().AddReadErrCallback([]() { + uiInterface.ThreadSafeMessageBox( + _("Error reading from database, shutting down.").translated, + "", CClientUIInterface::MSG_ERROR); + }); + + // If necessary, upgrade from older database format. + // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate + if (!chainstate->CoinsDB().Upgrade()) { + strLoadError = _("Error upgrading chainstate database").translated; + failed_chainstate_init = true; + break; + } - is_coinsview_empty = fReset || fReindexChainState || - ::ChainstateActive().CoinsTip().GetBestBlock().IsNull(); - if (!is_coinsview_empty) { - // LoadChainTip initializes the chain based on CoinsTip()'s best block - if (!::ChainstateActive().LoadChainTip(chainparams)) { - strLoadError = _("Error initializing block database").translated; + // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate + if (!chainstate->ReplayBlocks(chainparams)) { + strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; + failed_chainstate_init = true; break; } - assert(::ChainActive().Tip() != nullptr); + + // The on-disk coinsdb is now in a good state, create the cache + chainstate->InitCoinsCache(); + assert(chainstate->CanFlushToDisk()); + + if (!is_coinsview_empty(chainstate)) { + // LoadChainTip initializes the chain based on CoinsTip()'s best block + if (!chainstate->LoadChainTip(chainparams)) { + strLoadError = _("Error initializing block database").translated; + failed_chainstate_init = true; + break; // out of the per-chainstate loop + } + assert(chainstate->m_chain.Tip() != nullptr); + } + } + + if (failed_chainstate_init) { + break; // out of the chainstate activation do-while } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); @@ -1618,49 +1649,76 @@ bool AppInitMain(NodeContext& node) break; } - if (!fReset) { - // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. - // It both disconnects blocks based on ::ChainActive(), and drops block data in - // BlockIndex() based on lack of available witness data. - uiInterface.InitMessage(_("Rewinding blocks...").translated); - if (!RewindBlockIndex(chainparams)) { - strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain").translated; - break; + bool failed_rewind{false}; + // Can't hold cs_main while calling RewindBlockIndex, so retrieve the relevant + // chainstates beforehand. + for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + if (!fReset) { + // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. + // It both disconnects blocks based on the chainstate, and drops block data in + // BlockIndex() based on lack of available witness data. + uiInterface.InitMessage(_("Rewinding blocks...").translated); + if (!chainstate->RewindBlockIndex(chainparams)) { + strLoadError = _( + "Unable to rewind the database to a pre-fork state. " + "You will need to redownload the blockchain").translated; + failed_rewind = true; + break; // out of the per-chainstate loop + } } } + if (failed_rewind) { + break; // out of the chainstate activation do-while + } + + bool failed_verification = false; + try { LOCK(cs_main); - if (!is_coinsview_empty) { - uiInterface.InitMessage(_("Verifying blocks...").translated); - if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { - LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", - MIN_BLOCKS_TO_KEEP); - } - CBlockIndex* tip = ::ChainActive().Tip(); - RPCNotifyBlockChange(true, tip); - if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { - strLoadError = _("The block database contains a block which appears to be from the future. " - "This may be due to your computer's date and time being set incorrectly. " - "Only rebuild the block database if you are sure that your computer's date and time are correct").translated; - break; - } - - if (!CVerifyDB().VerifyDB(chainparams, &::ChainstateActive().CoinsDB(), gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), - gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { - strLoadError = _("Corrupted block database detected").translated; - break; + for (CChainState* chainstate : g_chainman.GetAll()) { + if (!is_coinsview_empty(chainstate)) { + uiInterface.InitMessage(_("Verifying blocks...").translated); + if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { + LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", + MIN_BLOCKS_TO_KEEP); + } + + const CBlockIndex* tip = chainstate->m_chain.Tip(); + RPCNotifyBlockChange(true, tip); + if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { + strLoadError = _("The block database contains a block which appears to be from the future. " + "This may be due to your computer's date and time being set incorrectly. " + "Only rebuild the block database if you are sure that your computer's date and time are correct").translated; + failed_verification = true; + break; + } + + // Only verify the DB of the active chainstate. This is fixed in later + // work when we allow VerifyDB to be parameterized by chainstate. + if (&::ChainstateActive() == chainstate && + !CVerifyDB().VerifyDB( + chainparams, &chainstate->CoinsDB(), + gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), + gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { + strLoadError = _("Corrupted block database detected").translated; + failed_verification = true; + break; + } } } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); strLoadError = _("Error opening block database").translated; + failed_verification = true; break; } - fLoaded = true; - LogPrintf(" block index %15dms\n", GetTimeMillis() - load_block_index_start_time); + if (!failed_verification) { + fLoaded = true; + LogPrintf(" block index %15dms\n", GetTimeMillis() - load_block_index_start_time); + } } while(false); if (!fLoaded && !ShutdownRequested()) { @@ -1724,8 +1782,11 @@ bool AppInitMain(NodeContext& node) LogPrintf("Unsetting NODE_NETWORK on prune mode\n"); nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK); if (!fReindex) { - uiInterface.InitMessage(_("Pruning blockstore...").translated); - ::ChainstateActive().PruneAndFlush(); + LOCK(cs_main); + for (CChainState* chainstate : g_chainman.GetAll()) { + uiInterface.InitMessage(_("Pruning blockstore...").translated); + chainstate->PruneAndFlush(); + } } } diff --git a/src/init.h b/src/init.h index f74ae5a47a..ef568b6f38 100644 --- a/src/init.h +++ b/src/init.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -54,9 +54,9 @@ bool AppInitLockDataDirectory(); bool AppInitMain(NodeContext& node); /** - * Setup the arguments for gArgs + * Register all arguments with the ArgsManager */ -void SetupServerArgs(); +void SetupServerArgs(NodeContext& node); /** Returns licensing information (for -version) */ std::string LicenseInfo(); diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index 0b3cd08e22..0e7641ae32 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -38,6 +38,21 @@ namespace interfaces { namespace { +bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock) +{ + if (!index) return false; + if (block.m_hash) *block.m_hash = index->GetBlockHash(); + if (block.m_height) *block.m_height = index->nHeight; + if (block.m_time) *block.m_time = index->GetBlockTime(); + if (block.m_max_time) *block.m_max_time = index->GetBlockTimeMax(); + if (block.m_mtp_time) *block.m_mtp_time = index->GetMedianTimePast(); + if (block.m_data) { + REVERSE_LOCK(lock); + if (!ReadBlockFromDisk(*block.m_data, index, Params().GetConsensus())) block.m_data->SetNull(); + } + return true; +} + class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> { Optional<int> getHeight() override @@ -65,20 +80,6 @@ class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> assert(block != nullptr); return block->GetBlockHash(); } - int64_t getBlockTime(int height) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; - assert(block != nullptr); - return block->GetBlockTime(); - } - int64_t getBlockMedianTimePast(int height) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; - assert(block != nullptr); - return block->GetMedianTimePast(); - } bool haveBlockOnDisk(int height) override { LockAssertion lock(::cs_main); @@ -95,20 +96,6 @@ class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> } return nullopt; } - Optional<int> findPruned(int start_height, Optional<int> stop_height) override - { - LockAssertion lock(::cs_main); - if (::fPruneMode) { - CBlockIndex* block = stop_height ? ::ChainActive()[*stop_height] : ::ChainActive().Tip(); - while (block && block->nHeight >= start_height) { - if ((block->nStatus & BLOCK_HAVE_DATA) == 0) { - return block->nHeight; - } - block = block->pprev; - } - } - return nullopt; - } Optional<int> findFork(const uint256& hash, Optional<int>* height) override { LockAssertion lock(::cs_main); @@ -247,26 +234,50 @@ public: std::unique_ptr<Chain::Lock> result = std::move(lock); // Temporary to avoid CWG 1579 return result; } - bool findBlock(const uint256& hash, CBlock* block, int64_t* time, int64_t* time_max) override + bool findBlock(const uint256& hash, const FoundBlock& block) override { - CBlockIndex* index; - { - LOCK(cs_main); - index = LookupBlockIndex(hash); - if (!index) { - return false; - } - if (time) { - *time = index->GetBlockTime(); - } - if (time_max) { - *time_max = index->GetBlockTimeMax(); + WAIT_LOCK(cs_main, lock); + return FillBlock(LookupBlockIndex(hash), block, lock); + } + bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override + { + WAIT_LOCK(cs_main, lock); + return FillBlock(ChainActive().FindEarliestAtLeast(min_time, min_height), block, lock); + } + bool findNextBlock(const uint256& block_hash, int block_height, const FoundBlock& next, bool* reorg) override { + WAIT_LOCK(cs_main, lock); + CBlockIndex* block = ChainActive()[block_height]; + if (block && block->GetBlockHash() != block_hash) block = nullptr; + if (reorg) *reorg = !block; + return FillBlock(block ? ChainActive()[block_height + 1] : nullptr, next, lock); + } + bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out) override + { + WAIT_LOCK(cs_main, lock); + if (const CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { + return FillBlock(ancestor, ancestor_out, lock); } } - if (block && !ReadBlockFromDisk(*block, index, Params().GetConsensus())) { - block->SetNull(); - } - return true; + return FillBlock(nullptr, ancestor_out, lock); + } + bool findAncestorByHash(const uint256& block_hash, const uint256& ancestor_hash, const FoundBlock& ancestor_out) override + { + WAIT_LOCK(cs_main, lock); + const CBlockIndex* block = LookupBlockIndex(block_hash); + const CBlockIndex* ancestor = LookupBlockIndex(ancestor_hash); + if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; + return FillBlock(ancestor, ancestor_out, lock); + } + bool findCommonAncestor(const uint256& block_hash1, const uint256& block_hash2, const FoundBlock& ancestor_out, const FoundBlock& block1_out, const FoundBlock& block2_out) override + { + WAIT_LOCK(cs_main, lock); + const CBlockIndex* block1 = LookupBlockIndex(block_hash1); + const CBlockIndex* block2 = LookupBlockIndex(block_hash2); + const CBlockIndex* ancestor = block1 && block2 ? LastCommonAncestor(block1, block2) : nullptr; + // Using & instead of && below to avoid short circuiting and leaving + // output uninitialized. + return FillBlock(ancestor, ancestor_out, lock) & FillBlock(block1, block1_out, lock) & FillBlock(block2, block2_out, lock); } void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override @@ -274,6 +285,25 @@ public: LOCK(cs_main); return GuessVerificationProgress(Params().TxData(), LookupBlockIndex(block_hash)); } + bool hasBlocks(const uint256& block_hash, int min_height, Optional<int> max_height) override + { + // hasBlocks returns true if all ancestors of block_hash in specified + // range have block data (are not pruned), false if any ancestors in + // specified range are missing data. + // + // For simplicity and robustness, min_height and max_height are only + // used to limit the range, and passing min_height that's too low or + // max_height that's too high will not crash or change the result. + LOCK(::cs_main); + if (CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (max_height && block->nHeight >= *max_height) block = block->GetAncestor(*max_height); + for (; block->nStatus & BLOCK_HAVE_DATA; block = block->pprev) { + // Check pprev to not segfault if min_height is too low + if (block->nHeight <= min_height || !block->pprev) return true; + } + } + return false; + } RBFTransactionState isRBFOptIn(const CTransaction& tx) override { LOCK(::mempool.cs); diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index e1bc9bbbf3..fb77c81cdc 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -30,6 +30,27 @@ namespace interfaces { class Handler; class Wallet; +//! Helper for findBlock to selectively return pieces of block data. +class FoundBlock +{ +public: + FoundBlock& hash(uint256& hash) { m_hash = &hash; return *this; } + FoundBlock& height(int& height) { m_height = &height; return *this; } + FoundBlock& time(int64_t& time) { m_time = &time; return *this; } + FoundBlock& maxTime(int64_t& max_time) { m_max_time = &max_time; return *this; } + FoundBlock& mtpTime(int64_t& mtp_time) { m_mtp_time = &mtp_time; return *this; } + //! Read block data from disk. If the block exists but doesn't have data + //! (for example due to pruning), the CBlock variable will be set to null. + FoundBlock& data(CBlock& data) { m_data = &data; return *this; } + + uint256* m_hash = nullptr; + int* m_height = nullptr; + int64_t* m_time = nullptr; + int64_t* m_max_time = nullptr; + int64_t* m_mtp_time = nullptr; + CBlock* m_data = nullptr; +}; + //! Interface giving clients (wallet processes, maybe other analysis tools in //! the future) ability to access to the chain state, receive notifications, //! estimate fees, and submit transactions. @@ -79,13 +100,6 @@ public: //! Get block hash. Height must be valid or this function will abort. virtual uint256 getBlockHash(int height) = 0; - //! Get block time. Height must be valid or this function will abort. - virtual int64_t getBlockTime(int height) = 0; - - //! Get block median time past. Height must be valid or this function - //! will abort. - virtual int64_t getBlockMedianTimePast(int height) = 0; - //! Check that the block is available on disk (i.e. has not been //! pruned), and contains transactions. virtual bool haveBlockOnDisk(int height) = 0; @@ -97,10 +111,6 @@ public: //! (to avoid the cost of a second lookup in case this information is needed.) virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) = 0; - //! Return height of last block in the specified range which is pruned, or - //! nullopt if no block in the range is pruned. Range is inclusive. - virtual Optional<int> findPruned(int start_height = 0, Optional<int> stop_height = nullopt) = 0; - //! Return height of the specified block if it is on the chain, otherwise //! return the height of the highest block on chain that's an ancestor //! of the specified block, or nullopt if there is no common ancestor. @@ -127,14 +137,36 @@ public: //! Return whether node has the block and optionally return block metadata //! or contents. - //! - //! If a block pointer is provided to retrieve the block contents, and the - //! block exists but doesn't have data (for example due to pruning), the - //! block will be empty and all fields set to null. - virtual bool findBlock(const uint256& hash, - CBlock* block = nullptr, - int64_t* time = nullptr, - int64_t* max_time = nullptr) = 0; + virtual bool findBlock(const uint256& hash, const FoundBlock& block={}) = 0; + + //! Find first block in the chain with timestamp >= the given time + //! and height >= than the given height, return false if there is no block + //! with a high enough timestamp and height. Optionally return block + //! information. + virtual bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block={}) = 0; + + //! Find next block if block is part of current chain. Also flag if + //! there was a reorg and the specified block hash is no longer in the + //! current chain, and optionally return block information. + virtual bool findNextBlock(const uint256& block_hash, int block_height, const FoundBlock& next={}, bool* reorg=nullptr) = 0; + + //! Find ancestor of block at specified height and optionally return + //! ancestor information. + virtual bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out={}) = 0; + + //! Return whether block descends from a specified ancestor, and + //! optionally return ancestor information. + virtual bool findAncestorByHash(const uint256& block_hash, + const uint256& ancestor_hash, + const FoundBlock& ancestor_out={}) = 0; + + //! Find most recent common ancestor between two blocks and optionally + //! return block information. + virtual bool findCommonAncestor(const uint256& block_hash1, + const uint256& block_hash2, + const FoundBlock& ancestor_out={}, + const FoundBlock& block1_out={}, + const FoundBlock& block2_out={}) = 0; //! Look up unspent output information. Returns coins in the mempool and in //! the current chain UTXO set. Iterates through all the keys in the map and @@ -145,6 +177,11 @@ public: //! the specified block hash are verified. virtual double guessVerificationProgress(const uint256& block_hash) = 0; + //! Return true if data is available for all blocks in the specified range + //! of blocks. This checks all blocks that are ancestors of block_hash in + //! the height range from min_height to max_height, inclusive. + virtual bool hasBlocks(const uint256& block_hash, int min_height = 0, Optional<int> max_height = {}) = 0; + //! Check if transaction is RBF opt in. virtual RBFTransactionState isRBFOptIn(const CTransaction& tx) = 0; @@ -283,6 +320,12 @@ public: //! Shut down client. virtual void stop() = 0; + + //! Set mock time. + virtual void setMockTime(int64_t time) = 0; + + //! Return interfaces for accessing wallets (if any). + virtual std::vector<std::unique_ptr<Wallet>> getWallets() = 0; }; //! Return implementation of Chain interface. diff --git a/src/interfaces/handler.cpp b/src/interfaces/handler.cpp index 95035c1b54..4134a4527f 100644 --- a/src/interfaces/handler.cpp +++ b/src/interfaces/handler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/interfaces/handler.h b/src/interfaces/handler.h index fbac3c6b71..11baf9dd65 100644 --- a/src/interfaces/handler.h +++ b/src/interfaces/handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 905173d20b..5ebbd61584 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -97,7 +97,7 @@ public: StopMapPort(); } } - void setupServerArgs() override { return SetupServerArgs(); } + void setupServerArgs() override { return SetupServerArgs(m_context); } bool getProxy(Network net, proxyType& proxy_info) override { return GetProxy(net, proxy_info); } size_t getNodeCount(CConnman::NumConnections flags) override { @@ -253,8 +253,9 @@ public: std::vector<std::unique_ptr<Wallet>> getWallets() override { std::vector<std::unique_ptr<Wallet>> wallets; - for (const std::shared_ptr<CWallet>& wallet : GetWallets()) { - wallets.emplace_back(MakeWallet(wallet)); + for (auto& client : m_context.chain_clients) { + auto client_wallets = client->getWallets(); + std::move(client_wallets.begin(), client_wallets.end(), std::back_inserter(wallets)); } return wallets; } diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index abce09ca4a..152917ac60 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,7 @@ #include <sync.h> #include <ui_interface.h> #include <uint256.h> +#include <util/check.h> #include <util/system.h> #include <wallet/feebumper.h> #include <wallet/fees.h> @@ -62,7 +63,7 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx) { WalletTxStatus result; - result.block_height = locked_chain.getBlockHeight(wtx.m_confirm.hashBlock).get_value_or(std::numeric_limits<int>::max()); + result.block_height = wtx.m_confirm.block_height > 0 ? wtx.m_confirm.block_height : std::numeric_limits<int>::max(); result.blocks_to_maturity = wtx.GetBlocksToMaturity(); result.depth_in_main_chain = wtx.GetDepthInMainChain(); result.time_received = wtx.nTimeReceived; @@ -318,13 +319,9 @@ public: if (mi == m_wallet->mapWallet.end()) { return false; } - if (Optional<int> height = locked_chain->getHeight()) { - num_blocks = *height; - block_time = locked_chain->getBlockTime(*height); - } else { - num_blocks = -1; - block_time = -1; - } + num_blocks = m_wallet->GetLastBlockHeight(); + block_time = -1; + CHECK_NONFATAL(m_wallet->chain().findBlock(m_wallet->GetLastBlockHash(), FoundBlock().time(block_time))); tx_status = MakeWalletTxStatus(*locked_chain, mi->second); return true; } @@ -373,12 +370,12 @@ public: { auto locked_chain = m_wallet->chain().lock(true /* try_lock */); if (!locked_chain) return false; - num_blocks = locked_chain->getHeight().get_value_or(-1); - if (!force && num_blocks == cached_num_blocks) return false; TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; } + num_blocks = m_wallet->GetLastBlockHeight(); + if (!force && num_blocks == cached_num_blocks) return false; balances = getBalances(); return true; } @@ -467,6 +464,7 @@ public: { RemoveWallet(m_wallet); } + bool isLegacy() override { return m_wallet->IsLegacy(); } std::unique_ptr<Handler> handleUnload(UnloadFn fn) override { return MakeHandler(m_wallet->NotifyUnload.connect(fn)); @@ -519,6 +517,15 @@ public: void start(CScheduler& scheduler) override { return StartWallets(scheduler); } void flush() override { return FlushWallets(); } void stop() override { return StopWallets(); } + void setMockTime(int64_t time) override { return SetMockTime(time); } + std::vector<std::unique_ptr<Wallet>> getWallets() override + { + std::vector<std::unique_ptr<Wallet>> wallets; + for (const auto& wallet : GetWallets()) { + wallets.emplace_back(MakeWallet(wallet)); + } + return wallets; + } ~WalletClientImpl() override { UnloadWallets(); } Chain& m_chain; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index dbb0912230..5d870c5e3d 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,6 @@ #include <functional> #include <map> #include <memory> -#include <psbt.h> #include <stdint.h> #include <string> #include <tuple> @@ -26,12 +25,13 @@ class CCoinControl; class CFeeRate; class CKey; class CWallet; -enum isminetype : unsigned int; enum class FeeReason; -typedef uint8_t isminefilter; - enum class OutputType; +enum class TransactionError; +enum isminetype : unsigned int; struct CRecipient; +struct PartiallySignedTransaction; +typedef uint8_t isminefilter; namespace interfaces { @@ -266,6 +266,9 @@ public: // Remove wallet. virtual void remove() = 0; + //! Return whether is a legacy wallet + virtual bool isLegacy() = 0; + //! Register handler for unload message. using UnloadFn = std::function<void()>; virtual std::unique_ptr<Handler> handleUnload(UnloadFn fn) = 0; diff --git a/src/logging.cpp b/src/logging.cpp index 6fd916b603..eb9da06d9b 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -182,30 +182,15 @@ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str) return false; } -std::string ListLogCategories() +std::vector<LogCategory> BCLog::Logger::LogCategoriesList() { - std::string ret; - int outcount = 0; + std::vector<LogCategory> ret; for (const CLogCategoryDesc& category_desc : LogCategories) { // Omit the special cases. if (category_desc.flag != BCLog::NONE && category_desc.flag != BCLog::ALL) { - if (outcount != 0) ret += ", "; - ret += category_desc.category; - outcount++; - } - } - return ret; -} - -std::vector<CLogCategoryActive> ListActiveLogCategories() -{ - std::vector<CLogCategoryActive> ret; - for (const CLogCategoryDesc& category_desc : LogCategories) { - // Omit the special cases. - if (category_desc.flag != BCLog::NONE && category_desc.flag != BCLog::ALL) { - CLogCategoryActive catActive; + LogCategory catActive; catActive.category = category_desc.category; - catActive.active = LogAcceptCategory(category_desc.flag); + catActive.active = WillLogCategory(category_desc.flag); ret.push_back(catActive); } } diff --git a/src/logging.h b/src/logging.h index b2fde1b9ea..ab07010316 100644 --- a/src/logging.h +++ b/src/logging.h @@ -8,6 +8,7 @@ #include <fs.h> #include <tinyformat.h> +#include <util/string.h> #include <atomic> #include <cstdint> @@ -24,8 +25,7 @@ extern const char * const DEFAULT_DEBUGLOGFILE; extern bool fLogIPs; -struct CLogCategoryActive -{ +struct LogCategory { std::string category; bool active; }; @@ -132,6 +132,13 @@ namespace BCLog { bool DisableCategory(const std::string& str); bool WillLogCategory(LogFlags category) const; + /** Returns a vector of the log categories */ + std::vector<LogCategory> LogCategoriesList(); + /** Returns a string with the log categories */ + std::string LogCategoriesString() + { + return Join(LogCategoriesList(), ", ", [&](const LogCategory& i) { return i.category; }); + }; bool DefaultShrinkDebugFile() const; }; @@ -146,12 +153,6 @@ static inline bool LogAcceptCategory(BCLog::LogFlags category) return LogInstance().WillLogCategory(category); } -/** Returns a string with the log categories. */ -std::string ListLogCategories(); - -/** Returns a vector of the active log categories. */ -std::vector<CLogCategoryActive> ListActiveLogCategories(); - /** Return true if str parses as a log category and set the flag */ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str); diff --git a/src/logging/timer.h b/src/logging/timer.h index 2b27c71080..21bb3d121e 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/memusage.h b/src/memusage.h index 24eb450465..a6e894129a 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/miner.cpp b/src/miner.cpp index 61d27d17c1..d9dcbe8a70 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -39,6 +39,17 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam return nNewTime - nOldTime; } +void RegenerateCommitments(CBlock& block) +{ + CMutableTransaction tx{*block.vtx.at(0)}; + tx.vout.erase(tx.vout.begin() + GetWitnessCommitmentIndex(block)); + block.vtx.at(0) = MakeTransactionRef(tx); + + GenerateCoinbaseCommitment(block, WITH_LOCK(cs_main, return LookupBlockIndex(block.hashPrevBlock)), Params().GetConsensus()); + + block.hashMerkleRoot = BlockMerkleRoot(block); +} + BlockAssembler::Options::Options() { blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; diff --git a/src/miner.h b/src/miner.h index cc8fc31a9f..69296f9078 100644 --- a/src/miner.h +++ b/src/miner.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -203,4 +203,7 @@ private: void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce); int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); +/** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ +void RegenerateCommitments(CBlock& block); + #endif // BITCOIN_MINER_H @@ -45,16 +45,10 @@ static const bool DEFAULT_WHITELISTRELAY = true; /** Default for -whitelistforcerelay. */ static const bool DEFAULT_WHITELISTFORCERELAY = false; -/** Time between pings automatically sent out for latency probing and keepalive (in seconds). */ -static const int PING_INTERVAL = 2 * 60; /** Time after which to disconnect, after waiting for a ping response (or inactivity). */ static const int TIMEOUT_INTERVAL = 20 * 60; /** Run the feeler connection loop once every 2 minutes or 120 seconds. **/ static const int FEELER_INTERVAL = 120; -/** The maximum number of entries in an 'inv' protocol message */ -static const unsigned int MAX_INV_SZ = 50000; -/** The maximum number of entries in a locator */ -static const unsigned int MAX_LOCATOR_SZ = 101; /** The maximum number of new addresses to accumulate before announcing. */ static const unsigned int MAX_ADDR_TO_SEND = 1000; /** Maximum length of incoming protocol messages (no message over 4 MB is currently acceptable). */ @@ -479,6 +473,7 @@ private: std::atomic<int64_t> m_next_send_inv_to_incoming{0}; friend struct CConnmanTest; + friend struct ConnmanTestMsg; }; void Discover(); void StartMapPort(); @@ -718,6 +713,8 @@ public: class CNode { friend class CConnman; + friend struct ConnmanTestMsg; + public: std::unique_ptr<TransportDeserializer> m_deserializer; std::unique_ptr<TransportSerializer> m_serializer; @@ -794,8 +791,8 @@ public: std::vector<CAddress> vAddrToSend; const std::unique_ptr<CRollingBloomFilter> m_addr_known; bool fGetAddr{false}; - int64_t nNextAddrSend GUARDED_BY(cs_sendProcessing){0}; - int64_t nNextLocalAddrSend GUARDED_BY(cs_sendProcessing){0}; + std::chrono::microseconds m_next_addr_send GUARDED_BY(cs_sendProcessing){0}; + std::chrono::microseconds m_next_local_addr_send GUARDED_BY(cs_sendProcessing){0}; bool IsAddrRelayPeer() const { return m_addr_known != nullptr; } @@ -806,14 +803,13 @@ public: RecursiveMutex cs_inventory; struct TxRelay { - TxRelay() { pfilter = MakeUnique<CBloomFilter>(); } mutable RecursiveMutex cs_filter; // We use fRelayTxes for two purposes - // a) it allows us to not relay tx invs before receiving the peer's version message // b) the peer may tell us in its version message that we should not relay tx invs // unless it loads a bloom filter. bool fRelayTxes GUARDED_BY(cs_filter){false}; - std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter); + std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter){nullptr}; mutable RecursiveMutex cs_tx_inventory; CRollingBloomFilter filterInventoryKnown GUARDED_BY(cs_tx_inventory){50000, 0.000001}; diff --git a/src/net_permissions.h b/src/net_permissions.h index ad74848347..962a2159fc 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/net_processing.cpp b/src/net_processing.cpp index ab430cbe19..26327ac6eb 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -64,6 +64,12 @@ static constexpr int STALE_RELAY_AGE_LIMIT = 30 * 24 * 60 * 60; /// Age after which a block is considered historical for purposes of rate /// limiting block relay. Set to one week, denominated in seconds. static constexpr int HISTORICAL_BLOCK_AGE = 7 * 24 * 60 * 60; +/** Time between pings automatically sent out for latency probing and keepalive (in seconds). */ +static const int PING_INTERVAL = 2 * 60; +/** The maximum number of entries in a locator */ +static const unsigned int MAX_LOCATOR_SZ = 101; +/** The maximum number of entries in an 'inv' protocol message */ +static const unsigned int MAX_INV_SZ = 50000; /** Maximum number of in-flight transactions from a peer */ static constexpr int32_t MAX_PEER_TX_IN_FLIGHT = 100; /** Maximum number of announced transactions from a peer */ @@ -80,7 +86,47 @@ static_assert(INBOUND_PEER_TX_DELAY >= MAX_GETDATA_RANDOM_DELAY, "To preserve security, MAX_GETDATA_RANDOM_DELAY should not exceed INBOUND_PEER_DELAY"); /** Limit to avoid sending big packets. Not used in processing incoming GETDATA for compatibility */ static const unsigned int MAX_GETDATA_SZ = 1000; - +/** Number of blocks that can be requested at any given time from a single peer. */ +static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; +/** Timeout in seconds during which a peer must stall block download progress before being disconnected. */ +static const unsigned int BLOCK_STALLING_TIMEOUT = 2; +/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends + * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ +static const unsigned int MAX_HEADERS_RESULTS = 2000; +/** Maximum depth of blocks we're willing to serve as compact blocks to peers + * when requested. For older blocks, a regular BLOCK response will be sent. */ +static const int MAX_CMPCTBLOCK_DEPTH = 5; +/** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ +static const int MAX_BLOCKTXN_DEPTH = 10; +/** Size of the "block download window": how far ahead of our current height do we fetch? + * Larger windows tolerate larger download speed differences between peer, but increase the potential + * degree of disordering of blocks on disk (which make reindexing and pruning harder). We'll probably + * want to make this a per-peer adaptive value at some point. */ +static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; +/** Block download timeout base, expressed in millionths of the block interval (i.e. 10 min) */ +static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; +/** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ +static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; +/** Maximum number of headers to announce when relaying blocks with headers message.*/ +static const unsigned int MAX_BLOCKS_TO_ANNOUNCE = 8; +/** Maximum number of unconnecting headers announcements before DoS score */ +static const int MAX_UNCONNECTING_HEADERS = 10; +/** Minimum blocks required to signal NODE_NETWORK_LIMITED */ +static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; +/** Average delay between local address broadcasts */ +static constexpr std::chrono::hours AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24}; +/** Average delay between peer address broadcasts */ +static constexpr std::chrono::seconds AVG_ADDRESS_BROADCAST_INTERVAL{30}; +/** Average delay between trickled inventory transmissions in seconds. + * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ +static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; +/** Maximum number of inventory items to send per transmission. + * Limits the impact of low-fee transaction floods. */ +static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; +/** Average delay between feefilter broadcasts in seconds. */ +static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; +/** Maximum feefilter broadcast delay after significant change. */ +static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; struct COrphanTx { // When modifying, adapt the copy of this definition in tests/DoS_tests. @@ -97,21 +143,6 @@ void EraseOrphansFor(NodeId peer); /** Increase a node's misbehavior score. */ void Misbehaving(NodeId nodeid, int howmuch, const std::string& message="") EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Average delay between local address broadcasts in seconds. */ -static constexpr unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 60 * 60; -/** Average delay between peer address broadcasts in seconds. */ -static const unsigned int AVG_ADDRESS_BROADCAST_INTERVAL = 30; -/** Average delay between trickled inventory transmissions in seconds. - * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ -static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; -/** Maximum number of inventory items to send per transmission. - * Limits the impact of low-fee transaction floods. */ -static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; -/** Average delay between feefilter broadcasts in seconds. */ -static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; -/** Maximum feefilter broadcast delay after significant change. */ -static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; - // Internal stuff namespace { /** Number of nodes with fSyncStarted. */ @@ -1365,7 +1396,7 @@ void RelayTransaction(const uint256& txid, const CConnman& connman) }); } -static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connman) +static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& connman) { unsigned int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) @@ -1373,7 +1404,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connma // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the m_addr_knowns of the chosen nodes prevent repeats uint64_t hashAddr = addr.GetHash(); - const CSipHasher hasher = connman->GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); + const CSipHasher hasher = connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24 * 60 * 60)); FastRandomContext insecure_rand; std::array<std::pair<uint64_t, CNode*>,2> best{{{0, nullptr}, {0, nullptr}}}; @@ -1398,7 +1429,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connma } }; - connman->ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); + connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); } void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, const CInv& inv, CConnman* connman) @@ -1911,9 +1942,9 @@ void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uin } } -bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) +bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) { - LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); + LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom->GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -1922,8 +1953,8 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR if (!(pfrom->GetLocalServices() & NODE_BLOOM) && - (strCommand == NetMsgType::FILTERLOAD || - strCommand == NetMsgType::FILTERADD)) + (msg_type == NetMsgType::FILTERLOAD || + msg_type == NetMsgType::FILTERADD)) { if (pfrom->nVersion >= NO_BLOOM_VERSION) { LOCK(cs_main); @@ -1935,7 +1966,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR } } - if (strCommand == NetMsgType::VERSION) { + if (msg_type == NetMsgType::VERSION) { // Each connection can only send one version message if (pfrom->nVersion != 0) { @@ -2107,7 +2138,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR // At this point, the outgoing message serialization version can't change. const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - if (strCommand == NetMsgType::VERACK) + if (msg_type == NetMsgType::VERACK) { pfrom->SetRecvVersion(std::min(pfrom->nVersion.load(), PROTOCOL_VERSION)); @@ -2152,7 +2183,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return false; } - if (strCommand == NetMsgType::ADDR) { + if (msg_type == NetMsgType::ADDR) { std::vector<CAddress> vAddr; vRecv >> vAddr; @@ -2192,7 +2223,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes - RelayAddress(addr, fReachable, connman); + RelayAddress(addr, fReachable, *connman); } // Do not store addresses outside our network if (fReachable) @@ -2206,13 +2237,13 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::SENDHEADERS) { + if (msg_type == NetMsgType::SENDHEADERS) { LOCK(cs_main); State(pfrom->GetId())->fPreferHeaders = true; return true; } - if (strCommand == NetMsgType::SENDCMPCT) { + if (msg_type == NetMsgType::SENDCMPCT) { bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 0; vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; @@ -2235,7 +2266,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::INV) { + if (msg_type == NetMsgType::INV) { std::vector<CInv> vInv; vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) @@ -2297,7 +2328,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETDATA) { + if (msg_type == NetMsgType::GETDATA) { std::vector<CInv> vInv; vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) @@ -2318,7 +2349,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETBLOCKS) { + if (msg_type == NetMsgType::GETBLOCKS) { CBlockLocator locator; uint256 hashStop; vRecv >> locator >> hashStop; @@ -2386,7 +2417,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETBLOCKTXN) { + if (msg_type == NetMsgType::GETBLOCKTXN) { BlockTransactionsRequest req; vRecv >> req; @@ -2435,7 +2466,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETHEADERS) { + if (msg_type == NetMsgType::GETHEADERS) { CBlockLocator locator; uint256 hashStop; vRecv >> locator >> hashStop; @@ -2502,7 +2533,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::TX) { + if (msg_type == NetMsgType::TX) { // Stop processing the transaction early if // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off // or if this peer is supposed to be a block-relay-only peer @@ -2644,7 +2675,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::CMPCTBLOCK) + if (msg_type == NetMsgType::CMPCTBLOCK) { // Ignore cmpctblock received while importing if (fImporting || fReindex) { @@ -2865,7 +2896,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::BLOCKTXN) + if (msg_type == NetMsgType::BLOCKTXN) { // Ignore blocktxn received while importing if (fImporting || fReindex) { @@ -2947,7 +2978,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::HEADERS) + if (msg_type == NetMsgType::HEADERS) { // Ignore headers received while importing if (fImporting || fReindex) { @@ -2973,7 +3004,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return ProcessHeadersMessage(pfrom, connman, mempool, headers, chainparams, /*via_compact_block=*/false); } - if (strCommand == NetMsgType::BLOCK) + if (msg_type == NetMsgType::BLOCK) { // Ignore block received while importing if (fImporting || fReindex) { @@ -3009,7 +3040,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETADDR) { + if (msg_type == NetMsgType::GETADDR) { // This asymmetric behavior for inbound and outbound connections was introduced // to prevent a fingerprinting attack: an attacker can send specific fake addresses // to users' AddrMan and later request them by sending getaddr messages. @@ -3043,7 +3074,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::MEMPOOL) { + if (msg_type == NetMsgType::MEMPOOL) { if (!(pfrom->GetLocalServices() & NODE_BLOOM) && !pfrom->HasPermission(PF_MEMPOOL)) { if (!pfrom->HasPermission(PF_NOBAN)) @@ -3071,7 +3102,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::PING) { + if (msg_type == NetMsgType::PING) { if (pfrom->nVersion > BIP0031_VERSION) { uint64_t nonce = 0; @@ -3092,7 +3123,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::PONG) { + if (msg_type == NetMsgType::PONG) { int64_t pingUsecEnd = nTimeReceived; uint64_t nonce = 0; size_t nAvail = vRecv.in_avail(); @@ -3148,7 +3179,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::FILTERLOAD) { + if (msg_type == NetMsgType::FILTERLOAD) { CBloomFilter filter; vRecv >> filter; @@ -3168,7 +3199,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::FILTERADD) { + if (msg_type == NetMsgType::FILTERADD) { std::vector<unsigned char> vData; vRecv >> vData; @@ -3192,19 +3223,19 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::FILTERCLEAR) { + if (msg_type == NetMsgType::FILTERCLEAR) { if (pfrom->m_tx_relay == nullptr) { return true; } LOCK(pfrom->m_tx_relay->cs_filter); if (pfrom->GetLocalServices() & NODE_BLOOM) { - pfrom->m_tx_relay->pfilter.reset(new CBloomFilter()); + pfrom->m_tx_relay->pfilter = nullptr; } pfrom->m_tx_relay->fRelayTxes = true; return true; } - if (strCommand == NetMsgType::FEEFILTER) { + if (msg_type == NetMsgType::FEEFILTER) { CAmount newFeeFilter = 0; vRecv >> newFeeFilter; if (MoneyRange(newFeeFilter)) { @@ -3217,7 +3248,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::NOTFOUND) { + if (msg_type == NetMsgType::NOTFOUND) { // Remove the NOTFOUND transactions from the peer LOCK(cs_main); CNodeState *state = State(pfrom->GetId()); @@ -3243,7 +3274,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR } // Ignore unknown commands for extensibility - LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->GetId()); + LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(msg_type), pfrom->GetId()); return true; } @@ -3338,7 +3369,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter LogPrint(BCLog::NET, "PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(msg.m_command), pfrom->GetId()); return fMoreWork; } - const std::string& strCommand = msg.m_command; + const std::string& msg_type = msg.m_command; // Message size unsigned int nMessageSize = msg.m_message_size; @@ -3348,7 +3379,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter if (!msg.m_valid_checksum) { LogPrint(BCLog::NET, "%s(%s, %u bytes): CHECKSUM ERROR peer=%d\n", __func__, - SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + SanitizeString(msg_type), nMessageSize, pfrom->GetId()); return fMoreWork; } @@ -3356,19 +3387,19 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fRet = false; try { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.m_time, chainparams, m_mempool, connman, m_banman, interruptMsgProc); + fRet = ProcessMessage(pfrom, msg_type, vRecv, msg.m_time, chainparams, m_mempool, connman, m_banman, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) fMoreWork = true; } catch (const std::exception& e) { - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what(), typeid(e).name()); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(msg_type), nMessageSize, e.what(), typeid(e).name()); } catch (...) { - LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(strCommand), nMessageSize); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(msg_type), nMessageSize); } if (!fRet) { - LogPrint(BCLog::NET, "%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + LogPrint(BCLog::NET, "%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(msg_type), nMessageSize, pfrom->GetId()); } LOCK(cs_main); @@ -3583,16 +3614,16 @@ bool PeerLogicValidation::SendMessages(CNode* pto) int64_t nNow = GetTimeMicros(); auto current_time = GetTime<std::chrono::microseconds>(); - if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { + if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->m_next_local_addr_send < current_time) { AdvertiseLocal(pto); - pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); + pto->m_next_local_addr_send = PoissonNextSend(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } // // Message: addr // - if (pto->IsAddrRelayPeer() && pto->nNextAddrSend < nNow) { - pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); + if (pto->IsAddrRelayPeer() && pto->m_next_addr_send < current_time) { + pto->m_next_addr_send = PoissonNextSend(current_time, AVG_ADDRESS_BROADCAST_INTERVAL); std::vector<CAddress> vAddr; vAddr.reserve(pto->vAddrToSend.size()); assert(pto->m_addr_known); diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 228caf74a9..6adb171abd 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/netaddress.h b/src/netaddress.h index b7381c1eb4..d0ab770379 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 641b2a5d9c..ec52a08ace 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/context.cpp b/src/node/context.cpp index 5b19a41bd4..0238aab0d9 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/context.h b/src/node/context.h index 1c592b456b..566ff170be 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <memory> #include <vector> +class ArgsManager; class BanMan; class CConnman; class CScheduler; @@ -33,6 +34,7 @@ struct NodeContext { CTxMemPool* mempool{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<PeerLogicValidation> peer_logic; std::unique_ptr<BanMan> banman; + ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<interfaces::Chain> chain; std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; std::unique_ptr<CScheduler> scheduler; diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index 5b16035f7d..c189018268 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/transaction.h b/src/node/transaction.h index a85dfb8ace..6491700d44 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,11 +6,19 @@ #define BITCOIN_NODE_TRANSACTION_H #include <attributes.h> +#include <policy/feerate.h> #include <primitives/transaction.h> #include <util/error.h> struct NodeContext; +/** Maximum fee rate for sendrawtransaction and testmempoolaccept RPC calls. + * Also used by the GUI when broadcasting a completed PSBT. + * By default, a transaction with a fee rate higher than this will be rejected + * by these RPCs and the GUI. This can be overridden with the maxfeerate argument. + */ +static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; + /** * Submit a transaction to the mempool and (optionally) relay it to all P2P peers. * diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 71b5cba01c..ea7a86d6d6 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/prevector.h b/src/prevector.h index 6d690e7f96..9f2f7ba2de 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 6e72c1f15c..e6183cf2f4 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/protocol.cpp b/src/protocol.cpp index bd3ed25a8a..a3e844e35b 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/psbt.cpp b/src/psbt.cpp index e6b6285652..ef9781817a 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/psbt.h b/src/psbt.h index dfba261961..af57994f3a 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -40,6 +40,10 @@ static constexpr uint8_t PSBT_OUT_BIP32_DERIVATION = 0x02; // as a 0 length key which indicates that this is the separator. The separator has no value. static constexpr uint8_t PSBT_SEPARATOR = 0x00; +// BIP 174 does not specify a maximum file size, but we set a limit anyway +// to prevent reading a stream indefinately and running out of memory. +const std::streamsize MAX_FILE_SIZE_PSBT = 100000000; // 100 MiB + /** A structure for PSBTs which contain per-input information */ struct PSBTInput { diff --git a/src/pubkey.h b/src/pubkey.h index 2fc92c9bc6..261842b7f7 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -219,6 +219,11 @@ struct CExtPubKey { a.pubkey == b.pubkey; } + friend bool operator!=(const CExtPubKey &a, const CExtPubKey &b) + { + return !(a == b); + } + void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const; void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); bool Derive(CExtPubKey& out, unsigned int nChild) const; diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 3869318fea..8110f4e895 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index 72f16bb09f..2739b21a9d 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index f01c506a1e..88d5b5811b 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 4313d6ee7f..8939b566f7 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -336,7 +336,7 @@ void BitcoinApplication::initializeResult(bool success) window->setClientModel(clientModel); #ifdef ENABLE_WALLET if (WalletModel::isWalletEnabled()) { - m_wallet_controller = new WalletController(m_node, platformStyle, optionsModel, this); + m_wallet_controller = new WalletController(*clientModel, platformStyle, this); window->setWalletController(m_wallet_controller); if (paymentServer) { paymentServer->setOptionsModel(optionsModel); @@ -381,7 +381,7 @@ void BitcoinApplication::shutdownResult() void BitcoinApplication::handleRunawayException(const QString &message) { - QMessageBox::critical(nullptr, "Runaway exception", BitcoinGUI::tr("A fatal error occurred. Bitcoin can no longer continue safely and will quit.") + QString("\n\n") + message); + QMessageBox::critical(nullptr, "Runaway exception", BitcoinGUI::tr("A fatal error occurred. %1 can no longer continue safely and will quit.").arg(PACKAGE_NAME) + QString("\n\n") + message); ::exit(EXIT_FAILURE); } diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 75f3e9bf45..71e4d59ea7 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -317,6 +317,8 @@ void BitcoinGUI::createActions() signMessageAction->setStatusTip(tr("Sign messages with your Bitcoin addresses to prove you own them")); verifyMessageAction = new QAction(tr("&Verify message..."), this); verifyMessageAction->setStatusTip(tr("Verify messages to ensure they were signed with specified Bitcoin addresses")); + m_load_psbt_action = new QAction(tr("Load PSBT..."), this); + m_load_psbt_action->setStatusTip(tr("Load Partially Signed Bitcoin Transaction")); openRPCConsoleAction = new QAction(tr("Node window"), this); openRPCConsoleAction->setStatusTip(tr("Open node debugging and diagnostic console")); @@ -366,6 +368,7 @@ void BitcoinGUI::createActions() connect(changePassphraseAction, &QAction::triggered, walletFrame, &WalletFrame::changePassphrase); connect(signMessageAction, &QAction::triggered, [this]{ showNormalIfMinimized(); }); connect(signMessageAction, &QAction::triggered, [this]{ gotoSignMessageTab(); }); + connect(m_load_psbt_action, &QAction::triggered, [this]{ gotoLoadPSBT(); }); connect(verifyMessageAction, &QAction::triggered, [this]{ showNormalIfMinimized(); }); connect(verifyMessageAction, &QAction::triggered, [this]{ gotoVerifyMessageTab(); }); connect(usedSendingAddressesAction, &QAction::triggered, walletFrame, &WalletFrame::usedSendingAddresses); @@ -438,6 +441,7 @@ void BitcoinGUI::createMenuBar() file->addAction(backupWalletAction); file->addAction(signMessageAction); file->addAction(verifyMessageAction); + file->addAction(m_load_psbt_action); file->addSeparator(); } file->addAction(quitAction); @@ -854,6 +858,10 @@ void BitcoinGUI::gotoVerifyMessageTab(QString addr) { if (walletFrame) walletFrame->gotoVerifyMessageTab(addr); } +void BitcoinGUI::gotoLoadPSBT() +{ + if (walletFrame) walletFrame->gotoLoadPSBT(); +} #endif // ENABLE_WALLET void BitcoinGUI::updateNetworkState() diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 809cf8b4ed..70366e12d1 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -135,6 +135,7 @@ private: QAction* usedReceivingAddressesAction = nullptr; QAction* signMessageAction = nullptr; QAction* verifyMessageAction = nullptr; + QAction* m_load_psbt_action = nullptr; QAction* aboutAction = nullptr; QAction* receiveCoinsAction = nullptr; QAction* receiveCoinsMenuAction = nullptr; @@ -270,6 +271,8 @@ public Q_SLOTS: void gotoSignMessageTab(QString addr = ""); /** Show Sign/Verify Message dialog and switch to verify message tab */ void gotoVerifyMessageTab(QString addr = ""); + /** Show load Partially Signed Bitcoin Transaction dialog */ + void gotoLoadPSBT(); /** Show open dialog */ void openClicked(); diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index a1ec3eaab1..b94fcc9865 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -105,6 +105,14 @@ int64_t ClientModel::getHeaderTipTime() const return cachedBestHeaderTime; } +int ClientModel::getNumBlocks() const +{ + if (m_cached_num_blocks == -1) { + m_cached_num_blocks = m_node.getNumBlocks(); + } + return m_cached_num_blocks; +} + void ClientModel::updateNumConnections(int numConnections) { Q_EMIT numConnectionsChanged(numConnections); @@ -241,6 +249,8 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, int heig // cache best headers time and height to reduce future cs_main locks clientmodel->cachedBestHeaderHeight = height; clientmodel->cachedBestHeaderTime = blockTime; + } else { + clientmodel->m_cached_num_blocks = height; } // During initial sync, block notifications, and header notifications from reindexing are both throttled. diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 79175e0af4..7ac4120a8f 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -56,6 +56,7 @@ public: //! Return number of connections, default is in- and outbound (total) int getNumConnections(unsigned int flags = CONNECTIONS_ALL) const; + int getNumBlocks() const; int getHeaderTipHeight() const; int64_t getHeaderTipTime() const; @@ -73,9 +74,10 @@ public: bool getProxyInfo(std::string& ip_port) const; - // caches for the best header + // caches for the best header, number of blocks mutable std::atomic<int> cachedBestHeaderHeight; mutable std::atomic<int64_t> cachedBestHeaderTime; + mutable std::atomic<int> m_cached_num_blocks{-1}; private: interfaces::Node& m_node; diff --git a/src/qt/coincontroltreewidget.h b/src/qt/coincontroltreewidget.h index 39dc9a5e9e..86af555385 100644 --- a/src/qt/coincontroltreewidget.h +++ b/src/qt/coincontroltreewidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index 8e6474b0d4..5056e487fc 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -60,3 +60,8 @@ bool CreateWalletDialog::isMakeBlankWalletChecked() const { return ui->blank_wallet_checkbox->isChecked(); } + +bool CreateWalletDialog::isDescriptorWalletChecked() const +{ + return ui->descriptor_checkbox->isChecked(); +} diff --git a/src/qt/createwalletdialog.h b/src/qt/createwalletdialog.h index 30766107b9..20cce937c8 100644 --- a/src/qt/createwalletdialog.h +++ b/src/qt/createwalletdialog.h @@ -27,6 +27,7 @@ public: bool isEncryptWalletChecked() const; bool isDisablePrivateKeysChecked() const; bool isMakeBlankWalletChecked() const; + bool isDescriptorWalletChecked() const; private: Ui::CreateWalletDialog *ui; diff --git a/src/qt/csvmodelwriter.h b/src/qt/csvmodelwriter.h index e443529335..ad247b6859 100644 --- a/src/qt/csvmodelwriter.h +++ b/src/qt/csvmodelwriter.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/forms/createwalletdialog.ui b/src/qt/forms/createwalletdialog.ui index e49bab8f3b..b592140dd7 100644 --- a/src/qt/forms/createwalletdialog.ui +++ b/src/qt/forms/createwalletdialog.ui @@ -7,7 +7,7 @@ <x>0</x> <y>0</y> <width>364</width> - <height>185</height> + <height>213</height> </rect> </property> <property name="windowTitle"> @@ -17,7 +17,7 @@ <property name="geometry"> <rect> <x>10</x> - <y>140</y> + <y>170</y> <width>341</width> <height>32</height> </rect> @@ -106,6 +106,22 @@ <string>Make Blank Wallet</string> </property> </widget> + <widget class="QCheckBox" name="descriptor_checkbox"> + <property name="geometry"> + <rect> + <x>20</x> + <y>140</y> + <width>171</width> + <height>22</height> + </rect> + </property> + <property name="toolTip"> + <string>Use descriptors for scriptPubKey management</string> + </property> + <property name="text"> + <string>Descriptor Wallet</string> + </property> + </widget> </widget> <tabstops> <tabstop>wallet_name_line_edit</tabstop> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 98dde1656a..a54b65961c 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/macnotificationhandler.mm b/src/qt/macnotificationhandler.mm index b16042e946..2d26a8383d 100644 --- a/src/qt/macnotificationhandler.mm +++ b/src/qt/macnotificationhandler.mm @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/main.cpp b/src/qt/main.cpp index 3dfd9e850e..6e772d58c5 100644 --- a/src/qt/main.cpp +++ b/src/qt/main.cpp @@ -1,10 +1,11 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/bitcoin.h> #include <util/translation.h> +#include <util/url.h> #include <QCoreApplication> @@ -15,5 +16,6 @@ extern const std::function<std::string(const char*)> G_TRANSLATION_FUN = [](const char* psz) { return QCoreApplication::translate("bitcoin-core", psz).toStdString(); }; +UrlDecodeFn* const URL_DECODE = urlDecode; int main(int argc, char* argv[]) { return GuiMain(argc, argv); } diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 6243a71c7d..49a1992803 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index c376921b72..e0ae1f9e92 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -161,20 +161,27 @@ void OverviewPage::setBalance(const interfaces::WalletBalances& balances) { int unit = walletModel->getOptionsModel()->getDisplayUnit(); m_balances = balances; - if (walletModel->wallet().privateKeysDisabled()) { - ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + if (walletModel->wallet().isLegacy()) { + if (walletModel->wallet().privateKeysDisabled()) { + ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + } else { + ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.balance, false, BitcoinUnits::separatorAlways)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_balance, false, BitcoinUnits::separatorAlways)); + ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_balance, false, BitcoinUnits::separatorAlways)); + ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchAvailable->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchPending->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + } } else { ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.balance, false, BitcoinUnits::separatorAlways)); ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_balance, false, BitcoinUnits::separatorAlways)); ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_balance, false, BitcoinUnits::separatorAlways)); ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchAvailable->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchPending->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); } // only show immature (newly mined) balance if it's non-zero, so as not to complicate things // for the non-mining users diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index a1fc791536..5220f8e138 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index cf45c5a08f..0e20b1d9dc 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/qvaluecombobox.h b/src/qt/qvaluecombobox.h index 5cca515079..bde9c0d1a6 100644 --- a/src/qt/qvaluecombobox.h +++ b/src/qt/qvaluecombobox.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 180550c5ae..5debded4ea 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -157,17 +157,40 @@ void ReceiveCoinsDialog::on_receiveButton_clicked() } } address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, "", address_type); - SendCoinsRecipient info(address, label, - ui->reqAmount->value(), ui->reqMessage->text()); - ReceiveRequestDialog *dialog = new ReceiveRequestDialog(this); - dialog->setAttribute(Qt::WA_DeleteOnClose); - dialog->setModel(model); - dialog->setInfo(info); - dialog->show(); - clear(); - /* Store request for later reference */ - model->getRecentRequestsTableModel()->addNewRequest(info); + switch(model->getAddressTableModel()->getEditStatus()) + { + case AddressTableModel::EditStatus::OK: { + // Success + SendCoinsRecipient info(address, label, + ui->reqAmount->value(), ui->reqMessage->text()); + ReceiveRequestDialog *dialog = new ReceiveRequestDialog(this); + dialog->setAttribute(Qt::WA_DeleteOnClose); + dialog->setModel(model); + dialog->setInfo(info); + dialog->show(); + + /* Store request for later reference */ + model->getRecentRequestsTableModel()->addNewRequest(info); + break; + } + case AddressTableModel::EditStatus::WALLET_UNLOCK_FAILURE: + QMessageBox::critical(this, windowTitle(), + tr("Could not unlock wallet."), + QMessageBox::Ok, QMessageBox::Ok); + break; + case AddressTableModel::EditStatus::KEY_GENERATION_FAILURE: + QMessageBox::critical(this, windowTitle(), + tr("Could not generate new %1 address").arg(QString::fromStdString(FormatOutputType(address_type))), + QMessageBox::Ok, QMessageBox::Ok); + break; + // These aren't valid return values for our action + case AddressTableModel::EditStatus::INVALID_ADDRESS: + case AddressTableModel::EditStatus::DUPLICATE_ADDRESS: + case AddressTableModel::EditStatus::NO_CHANGES: + assert(false); + } + clear(); } void ReceiveCoinsDialog::on_recentRequestsView_doubleClicked(const QModelIndex &index) diff --git a/src/qt/res/icons/bitcoin.ico b/src/qt/res/icons/bitcoin.ico Binary files differindex 8f5045015d..8f5045015d 100755..100644 --- a/src/qt/res/icons/bitcoin.ico +++ b/src/qt/res/icons/bitcoin.ico diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 0ffdc892c5..18a7e9b021 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index a8c82aaf6c..7b389a48d6 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -219,11 +219,8 @@ SendCoinsDialog::~SendCoinsDialog() delete ui; } -void SendCoinsDialog::on_sendButton_clicked() +bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informative_text, QString& detailed_text) { - if(!model || !model->getOptionsModel()) - return; - QList<SendCoinsRecipient> recipients; bool valid = true; @@ -246,7 +243,7 @@ void SendCoinsDialog::on_sendButton_clicked() if(!valid || recipients.isEmpty()) { - return; + return false; } fNewRecipientAllowed = false; @@ -255,11 +252,11 @@ void SendCoinsDialog::on_sendButton_clicked() { // Unlock wallet was cancelled fNewRecipientAllowed = true; - return; + return false; } // prepare transaction for getting txFee earlier - WalletModelTransaction currentTransaction(recipients); + m_current_transaction = MakeUnique<WalletModelTransaction>(recipients); WalletModel::SendCoinsReturn prepareStatus; // Always use a CCoinControl instance, use the CoinControlDialog instance if CoinControl has been enabled @@ -269,22 +266,20 @@ void SendCoinsDialog::on_sendButton_clicked() updateCoinControlState(ctrl); - prepareStatus = model->prepareTransaction(currentTransaction, ctrl); + prepareStatus = model->prepareTransaction(*m_current_transaction, ctrl); // process prepareStatus and on error generate message shown to user processSendCoinsReturn(prepareStatus, - BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), currentTransaction.getTransactionFee())); + BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), m_current_transaction->getTransactionFee())); if(prepareStatus.status != WalletModel::OK) { fNewRecipientAllowed = true; - return; + return false; } - CAmount txFee = currentTransaction.getTransactionFee(); - - // Format confirmation message + CAmount txFee = m_current_transaction->getTransactionFee(); QStringList formatted; - for (const SendCoinsRecipient &rcp : currentTransaction.getRecipients()) + for (const SendCoinsRecipient &rcp : m_current_transaction->getRecipients()) { // generate amount string with wallet name in case of multiwallet QString amount = BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), rcp.amount); @@ -311,72 +306,82 @@ void SendCoinsDialog::on_sendButton_clicked() formatted.append(recipientElement); } - QString questionString; if (model->wallet().privateKeysDisabled()) { - questionString.append(tr("Do you want to draft this transaction?")); + question_string.append(tr("Do you want to draft this transaction?")); } else { - questionString.append(tr("Are you sure you want to send?")); + question_string.append(tr("Are you sure you want to send?")); } - questionString.append("<br /><span style='font-size:10pt;'>"); + question_string.append("<br /><span style='font-size:10pt;'>"); if (model->wallet().privateKeysDisabled()) { - questionString.append(tr("Please, review your transaction proposal. This will produce a Partially Signed Bitcoin Transaction (PSBT) which you can copy and then sign with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); + question_string.append(tr("Please, review your transaction proposal. This will produce a Partially Signed Bitcoin Transaction (PSBT) which you can save or copy and then sign with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); } else { - questionString.append(tr("Please, review your transaction.")); + question_string.append(tr("Please, review your transaction.")); } - questionString.append("</span>%1"); + question_string.append("</span>%1"); if(txFee > 0) { // append fee string if a fee is required - questionString.append("<hr /><b>"); - questionString.append(tr("Transaction fee")); - questionString.append("</b>"); + question_string.append("<hr /><b>"); + question_string.append(tr("Transaction fee")); + question_string.append("</b>"); // append transaction size - questionString.append(" (" + QString::number((double)currentTransaction.getTransactionSize() / 1000) + " kB): "); + question_string.append(" (" + QString::number((double)m_current_transaction->getTransactionSize() / 1000) + " kB): "); // append transaction fee value - questionString.append("<span style='color:#aa0000; font-weight:bold;'>"); - questionString.append(BitcoinUnits::formatHtmlWithUnit(model->getOptionsModel()->getDisplayUnit(), txFee)); - questionString.append("</span><br />"); + question_string.append("<span style='color:#aa0000; font-weight:bold;'>"); + question_string.append(BitcoinUnits::formatHtmlWithUnit(model->getOptionsModel()->getDisplayUnit(), txFee)); + question_string.append("</span><br />"); // append RBF message according to transaction's signalling - questionString.append("<span style='font-size:10pt; font-weight:normal;'>"); + question_string.append("<span style='font-size:10pt; font-weight:normal;'>"); if (ui->optInRBF->isChecked()) { - questionString.append(tr("You can increase the fee later (signals Replace-By-Fee, BIP-125).")); + question_string.append(tr("You can increase the fee later (signals Replace-By-Fee, BIP-125).")); } else { - questionString.append(tr("Not signalling Replace-By-Fee, BIP-125.")); + question_string.append(tr("Not signalling Replace-By-Fee, BIP-125.")); } - questionString.append("</span>"); + question_string.append("</span>"); } // add total amount in all subdivision units - questionString.append("<hr />"); - CAmount totalAmount = currentTransaction.getTotalTransactionAmount() + txFee; + question_string.append("<hr />"); + CAmount totalAmount = m_current_transaction->getTotalTransactionAmount() + txFee; QStringList alternativeUnits; for (const BitcoinUnits::Unit u : BitcoinUnits::availableUnits()) { if(u != model->getOptionsModel()->getDisplayUnit()) alternativeUnits.append(BitcoinUnits::formatHtmlWithUnit(u, totalAmount)); } - questionString.append(QString("<b>%1</b>: <b>%2</b>").arg(tr("Total Amount")) + question_string.append(QString("<b>%1</b>: <b>%2</b>").arg(tr("Total Amount")) .arg(BitcoinUnits::formatHtmlWithUnit(model->getOptionsModel()->getDisplayUnit(), totalAmount))); - questionString.append(QString("<br /><span style='font-size:10pt; font-weight:normal;'>(=%1)</span>") + question_string.append(QString("<br /><span style='font-size:10pt; font-weight:normal;'>(=%1)</span>") .arg(alternativeUnits.join(" " + tr("or") + " "))); - QString informative_text; - QString detailed_text; if (formatted.size() > 1) { - questionString = questionString.arg(""); + question_string = question_string.arg(""); informative_text = tr("To review recipient list click \"Show Details...\""); detailed_text = formatted.join("\n\n"); } else { - questionString = questionString.arg("<br /><br />" + formatted.at(0)); + question_string = question_string.arg("<br /><br />" + formatted.at(0)); } + + return true; +} + +void SendCoinsDialog::on_sendButton_clicked() +{ + if(!model || !model->getOptionsModel()) + return; + + QString question_string, informative_text, detailed_text; + if (!PrepareSendText(question_string, informative_text, detailed_text)) return; + assert(m_current_transaction); + const QString confirmation = model->wallet().privateKeysDisabled() ? tr("Confirm transaction proposal") : tr("Confirm send coins"); - const QString confirmButtonText = model->wallet().privateKeysDisabled() ? tr("Copy PSBT to clipboard") : tr("Send"); - SendConfirmationDialog confirmationDialog(confirmation, questionString, informative_text, detailed_text, SEND_CONFIRM_DELAY, confirmButtonText, this); + const QString confirmButtonText = model->wallet().privateKeysDisabled() ? tr("Create Unsigned") : tr("Send"); + SendConfirmationDialog confirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, confirmButtonText, this); confirmationDialog.exec(); QMessageBox::StandardButton retval = static_cast<QMessageBox::StandardButton>(confirmationDialog.result()); @@ -388,7 +393,7 @@ void SendCoinsDialog::on_sendButton_clicked() bool send_failure = false; if (model->wallet().privateKeysDisabled()) { - CMutableTransaction mtx = CMutableTransaction{*(currentTransaction.getWtx())}; + CMutableTransaction mtx = CMutableTransaction{*(m_current_transaction->getWtx())}; PartiallySignedTransaction psbtx(mtx); bool complete = false; const TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, psbtx, complete); @@ -398,15 +403,51 @@ void SendCoinsDialog::on_sendButton_clicked() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); - Q_EMIT message(tr("PSBT copied"), "Copied to clipboard", CClientUIInterface::MSG_INFORMATION); + QMessageBox msgBox; + msgBox.setText("Unsigned Transaction"); + msgBox.setInformativeText("The PSBT has been copied to the clipboard. You can also save it."); + msgBox.setStandardButtons(QMessageBox::Save | QMessageBox::Discard); + msgBox.setDefaultButton(QMessageBox::Discard); + switch (msgBox.exec()) { + case QMessageBox::Save: { + QString selectedFilter; + QString fileNameSuggestion = ""; + bool first = true; + for (const SendCoinsRecipient &rcp : m_current_transaction->getRecipients()) { + if (!first) { + fileNameSuggestion.append(" - "); + } + QString labelOrAddress = rcp.label.isEmpty() ? rcp.address : rcp.label; + QString amount = BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), rcp.amount); + fileNameSuggestion.append(labelOrAddress + "-" + amount); + first = false; + } + fileNameSuggestion.append(".psbt"); + QString filename = GUIUtil::getSaveFileName(this, + tr("Save Transaction Data"), fileNameSuggestion, + tr("Partially Signed Transaction (Binary) (*.psbt)"), &selectedFilter); + if (filename.isEmpty()) { + return; + } + std::ofstream out(filename.toLocal8Bit().data()); + out << ssTx.str(); + out.close(); + Q_EMIT message(tr("PSBT saved"), "PSBT saved to disk", CClientUIInterface::MSG_INFORMATION); + break; + } + case QMessageBox::Discard: + break; + default: + assert(false); + } } else { // now send the prepared transaction - WalletModel::SendCoinsReturn sendStatus = model->sendCoins(currentTransaction); + WalletModel::SendCoinsReturn sendStatus = model->sendCoins(*m_current_transaction); // process sendStatus and on error generate message shown to user processSendCoinsReturn(sendStatus); if (sendStatus.status == WalletModel::OK) { - Q_EMIT coinsSent(currentTransaction.getWtx()->GetHash()); + Q_EMIT coinsSent(m_current_transaction->getWtx()->GetHash()); } else { send_failure = true; } @@ -417,10 +458,13 @@ void SendCoinsDialog::on_sendButton_clicked() coinControlUpdateLabels(); } fNewRecipientAllowed = true; + m_current_transaction.reset(); } void SendCoinsDialog::clear() { + m_current_transaction.reset(); + // Clear coin control settings CoinControlDialog::coinControl()->UnSelectAll(); ui->checkBoxCoinControlChange->setChecked(false); diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 86422c4030..36bc2a846b 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -60,6 +60,7 @@ private: Ui::SendCoinsDialog *ui; ClientModel *clientModel; WalletModel *model; + std::unique_ptr<WalletModelTransaction> m_current_transaction; bool fNewRecipientAllowed; bool fFeeMinimized; const PlatformStyle *platformStyle; @@ -69,6 +70,8 @@ private: // Additional parameter msgArg can be used via .arg(msgArg). void processSendCoinsReturn(const WalletModel::SendCoinsReturn &sendCoinsReturn, const QString &msgArg = QString()); void minimizeFeeSection(bool fMinimize); + // Format confirmation message + bool PrepareSendText(QString& question_string, QString& informative_text, QString& detailed_text); void updateFeeMinimizedLabel(); // Update the passed in CCoinControl with state from the GUI void updateCoinControlState(CCoinControl& ctrl); diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 4552753bf6..67c2c71e9c 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index e4ffa6cd9a..d8fe4a9238 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 042387286a..476128520c 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <interfaces/chain.h> #include <interfaces/node.h> +#include <qt/clientmodel.h> #include <qt/editaddressdialog.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> @@ -106,8 +107,9 @@ void TestAddAddressesToSendBook(interfaces::Node& node) // Initialize relevant QT models. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); OptionsModel optionsModel(node); + ClientModel clientModel(node, &optionsModel); AddWallet(wallet); - WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); RemoveWallet(wallet); EditAddressDialog editAddressDialog(EditAddressDialog::NewSendingAddress); editAddressDialog.setModel(walletModel.getAddressTableModel()); diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 14a75b23f3..f88d57c716 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -82,6 +82,7 @@ void AppTests::appTests() // Reset global state to avoid interfering with later tests. AbortShutdown(); UnloadBlockIndex(); + WITH_LOCK(::cs_main, g_chainman.Reset()); } //! Entry point for BitcoinGUI tests. diff --git a/src/qt/test/apptests.h b/src/qt/test/apptests.h index d16c9fe487..f0a555005d 100644 --- a/src/qt/test/apptests.h +++ b/src/qt/test/apptests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/util.cpp b/src/qt/test/util.cpp index e09f0ad77d..759832381b 100644 --- a/src/qt/test/util.cpp +++ b/src/qt/test/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/util.h b/src/qt/test/util.h index 763847606a..950314da3b 100644 --- a/src/qt/test/util.h +++ b/src/qt/test/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index c1a0f63f73..94a1e0a63d 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <interfaces/chain.h> #include <interfaces/node.h> #include <qt/bitcoinamountfield.h> +#include <qt/clientmodel.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> #include <qt/qvalidatedlineedit.h> @@ -151,12 +152,9 @@ void TestGUI(interfaces::Node& node) wallet->SetLastBlockProcessed(105, ::ChainActive().Tip()->GetBlockHash()); } { - auto locked_chain = wallet->chain().lock(); - LockAssertion lock(::cs_main); - - WalletRescanReserver reserver(wallet.get()); + WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(locked_chain->getBlockHash(0), {} /* stop_block */, reserver, true /* fUpdate */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(Params().GetConsensus().hashGenesisBlock, 0 /* block height */, {} /* max height */, reserver, true /* fUpdate */); QCOMPARE(result.status, CWallet::ScanResult::SUCCESS); QCOMPARE(result.last_scanned_block, ::ChainActive().Tip()->GetBlockHash()); QVERIFY(result.last_failed_block.IsNull()); @@ -168,8 +166,9 @@ void TestGUI(interfaces::Node& node) SendCoinsDialog sendCoinsDialog(platformStyle.get()); TransactionView transactionView(platformStyle.get()); OptionsModel optionsModel(node); + ClientModel clientModel(node, &optionsModel); AddWallet(wallet); - WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); RemoveWallet(wallet); sendCoinsDialog.setModel(&walletModel); transactionView.setModel(&walletModel); diff --git a/src/qt/trafficgraphwidget.h b/src/qt/trafficgraphwidget.h index af5890ba24..7e8bfb2337 100644 --- a/src/qt/trafficgraphwidget.h +++ b/src/qt/trafficgraphwidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiondescdialog.h b/src/qt/transactiondescdialog.h index 74e34cde87..3204b22657 100644 --- a/src/qt/transactiondescdialog.h +++ b/src/qt/transactiondescdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 64e9c856db..18554aef1f 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -5,6 +5,7 @@ #include <qt/transactiontablemodel.h> #include <qt/addresstablemodel.h> +#include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> @@ -175,7 +176,7 @@ public: return cachedWallet.size(); } - TransactionRecord *index(interfaces::Wallet& wallet, int idx) + TransactionRecord *index(interfaces::Wallet& wallet, const int cur_num_blocks, const int idx) { if(idx >= 0 && idx < cachedWallet.size()) { @@ -191,7 +192,7 @@ public: interfaces::WalletTxStatus wtx; int numBlocks; int64_t block_time; - if (wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time) && rec->statusUpdateNeeded(numBlocks)) { + if (rec->statusUpdateNeeded(cur_num_blocks) && wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time)) { rec->updateStatus(wtx, numBlocks, block_time); } return rec; @@ -663,10 +664,10 @@ QVariant TransactionTableModel::headerData(int section, Qt::Orientation orientat QModelIndex TransactionTableModel::index(int row, int column, const QModelIndex &parent) const { Q_UNUSED(parent); - TransactionRecord *data = priv->index(walletModel->wallet(), row); + TransactionRecord *data = priv->index(walletModel->wallet(), walletModel->clientModel().getNumBlocks(), row); if(data) { - return createIndex(row, column, priv->index(walletModel->wallet(), row)); + return createIndex(row, column, data); } return QModelIndex(); } diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 3101fb01c3..d16feba1bb 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index 833b86fd3e..b6a42d3d9d 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index f076b5ba61..9c1488fb0e 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -1,10 +1,11 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/walletcontroller.h> #include <qt/askpassphrasedialog.h> +#include <qt/clientmodel.h> #include <qt/createwalletdialog.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> @@ -24,13 +25,14 @@ #include <QTimer> #include <QWindow> -WalletController::WalletController(interfaces::Node& node, const PlatformStyle* platform_style, OptionsModel* options_model, QObject* parent) +WalletController::WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent) : QObject(parent) , m_activity_thread(new QThread(this)) , m_activity_worker(new QObject) - , m_node(node) + , m_client_model(client_model) + , m_node(client_model.node()) , m_platform_style(platform_style) - , m_options_model(options_model) + , m_options_model(client_model.getOptionsModel()) { m_handler_load_wallet = m_node.handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { getOrCreateWallet(std::move(wallet)); @@ -104,7 +106,7 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal } // Instantiate model and register it. - WalletModel* wallet_model = new WalletModel(std::move(wallet), m_node, m_platform_style, m_options_model, nullptr); + WalletModel* wallet_model = new WalletModel(std::move(wallet), m_client_model, m_platform_style, nullptr); // Handler callback runs in a different thread so fix wallet model thread affinity. wallet_model->moveToThread(thread()); wallet_model->setParent(this); @@ -224,6 +226,9 @@ void CreateWalletActivity::createWallet() if (m_create_wallet_dialog->isMakeBlankWalletChecked()) { flags |= WALLET_FLAG_BLANK_WALLET; } + if (m_create_wallet_dialog->isDescriptorWalletChecked()) { + flags |= WALLET_FLAG_DESCRIPTORS; + } QTimer::singleShot(500, worker(), [this, name, flags] { WalletCreationStatus status; diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index f30eb25308..3808b7d8bf 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,6 +21,7 @@ #include <QTimer> #include <QString> +class ClientModel; class OptionsModel; class PlatformStyle; class WalletModel; @@ -47,7 +48,7 @@ class WalletController : public QObject void removeAndDeleteWallet(WalletModel* wallet_model); public: - WalletController(interfaces::Node& node, const PlatformStyle* platform_style, OptionsModel* options_model, QObject* parent); + WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent); ~WalletController(); //! Returns wallet models currently open. @@ -70,6 +71,7 @@ Q_SIGNALS: private: QThread* const m_activity_thread; QObject* const m_activity_worker; + ClientModel& m_client_model; interfaces::Node& m_node; const PlatformStyle* const m_platform_style; OptionsModel* const m_options_model; diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index dac3326cc4..02a9583ae9 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -163,6 +163,14 @@ void WalletFrame::gotoVerifyMessageTab(QString addr) walletView->gotoVerifyMessageTab(addr); } +void WalletFrame::gotoLoadPSBT() +{ + WalletView *walletView = currentWalletView(); + if (walletView) { + walletView->gotoLoadPSBT(); + } +} + void WalletFrame::encryptWallet(bool status) { WalletView *walletView = currentWalletView(); diff --git a/src/qt/walletframe.h b/src/qt/walletframe.h index 20fad08b0e..d90ade5005 100644 --- a/src/qt/walletframe.h +++ b/src/qt/walletframe.h @@ -78,6 +78,9 @@ public Q_SLOTS: /** Show Sign/Verify Message dialog and switch to verify message tab */ void gotoVerifyMessageTab(QString addr = ""); + /** Load Partially Signed Bitcoin Transaction */ + void gotoLoadPSBT(); + /** Encrypt the wallet */ void encryptWallet(bool status); /** Backup the wallet */ diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 608797d6ad..e7581cd64e 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,6 +9,7 @@ #include <qt/walletmodel.h> #include <qt/addresstablemodel.h> +#include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> @@ -20,6 +21,7 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <key_io.h> +#include <psbt.h> #include <ui_interface.h> #include <util/system.h> // for GetBoolArg #include <wallet/coincontrol.h> @@ -33,8 +35,13 @@ #include <QTimer> -WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, interfaces::Node& node, const PlatformStyle *platformStyle, OptionsModel *_optionsModel, QObject *parent) : - QObject(parent), m_wallet(std::move(wallet)), m_node(node), optionsModel(_optionsModel), addressTableModel(nullptr), +WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent) : + QObject(parent), + m_wallet(std::move(wallet)), + m_client_model(client_model), + m_node(client_model.node()), + optionsModel(client_model.getOptionsModel()), + addressTableModel(nullptr), transactionTableModel(nullptr), recentRequestsTableModel(nullptr), cachedEncryptionStatus(Unencrypted), diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 7936014af9..07004b7c6b 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -24,6 +24,7 @@ enum class OutputType; class AddressTableModel; +class ClientModel; class OptionsModel; class PlatformStyle; class RecentRequestsTableModel; @@ -52,7 +53,7 @@ class WalletModel : public QObject Q_OBJECT public: - explicit WalletModel(std::unique_ptr<interfaces::Wallet> wallet, interfaces::Node& node, const PlatformStyle *platformStyle, OptionsModel *optionsModel, QObject *parent = nullptr); + explicit WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent = nullptr); ~WalletModel(); enum StatusCode // Returned by sendCoins @@ -143,6 +144,7 @@ public: interfaces::Node& node() const { return m_node; } interfaces::Wallet& wallet() const { return *m_wallet; } + ClientModel& clientModel() const { return m_client_model; } QString getWalletName() const; QString getDisplayName() const; @@ -159,6 +161,7 @@ private: std::unique_ptr<interfaces::Handler> m_handler_show_progress; std::unique_ptr<interfaces::Handler> m_handler_watch_only_changed; std::unique_ptr<interfaces::Handler> m_handler_can_get_addrs_changed; + ClientModel& m_client_model; interfaces::Node& m_node; bool fHaveWatchOnly; diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index bdcb82e06b..5d9b420df7 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -1,9 +1,12 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/walletview.h> +#include <node/psbt.h> +#include <node/transaction.h> +#include <policy/policy.h> #include <qt/addressbookpage.h> #include <qt/askpassphrasedialog.h> #include <qt/clientmodel.h> @@ -20,6 +23,7 @@ #include <interfaces/node.h> #include <ui_interface.h> +#include <util/strencodings.h> #include <QAction> #include <QActionGroup> @@ -197,6 +201,80 @@ void WalletView::gotoVerifyMessageTab(QString addr) signVerifyMessageDialog->setAddress_VM(addr); } +void WalletView::gotoLoadPSBT() +{ + QString filename = GUIUtil::getOpenFileName(this, + tr("Load Transaction Data"), QString(), + tr("Partially Signed Transaction (*.psbt)"), nullptr); + if (filename.isEmpty()) return; + if (GetFileSize(filename.toLocal8Bit().data(), MAX_FILE_SIZE_PSBT) == MAX_FILE_SIZE_PSBT) { + Q_EMIT message(tr("Error"), tr("PSBT file must be smaller than 100 MiB"), CClientUIInterface::MSG_ERROR); + return; + } + std::ifstream in(filename.toLocal8Bit().data(), std::ios::binary); + std::string data(std::istreambuf_iterator<char>{in}, {}); + + std::string error; + PartiallySignedTransaction psbtx; + if (!DecodeRawPSBT(psbtx, data, error)) { + Q_EMIT message(tr("Error"), tr("Unable to decode PSBT file") + "\n" + QString::fromStdString(error), CClientUIInterface::MSG_ERROR); + return; + } + + CMutableTransaction mtx; + bool complete = false; + PSBTAnalysis analysis = AnalyzePSBT(psbtx); + QMessageBox msgBox; + msgBox.setText("PSBT"); + switch (analysis.next) { + case PSBTRole::CREATOR: + case PSBTRole::UPDATER: + msgBox.setInformativeText("PSBT is incomplete. Copy to clipboard for manual inspection?"); + break; + case PSBTRole::SIGNER: + msgBox.setInformativeText("Transaction needs more signatures. Copy to clipboard?"); + break; + case PSBTRole::FINALIZER: + case PSBTRole::EXTRACTOR: + complete = FinalizeAndExtractPSBT(psbtx, mtx); + if (complete) { + msgBox.setInformativeText(tr("Would you like to send this transaction?")); + } else { + // The analyzer missed something, e.g. if there are final_scriptSig/final_scriptWitness + // but with invalid signatures. + msgBox.setInformativeText(tr("There was an unexpected problem processing the PSBT. Copy to clipboard for manual inspection?")); + } + } + + msgBox.setStandardButtons(QMessageBox::Yes | QMessageBox::Cancel); + switch (msgBox.exec()) { + case QMessageBox::Yes: { + if (complete) { + std::string err_string; + CTransactionRef tx = MakeTransactionRef(mtx); + + TransactionError result = BroadcastTransaction(*clientModel->node().context(), tx, err_string, DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK(), /* relay */ true, /* wait_callback */ false); + if (result == TransactionError::OK) { + Q_EMIT message(tr("Success"), tr("Broadcasted transaction sucessfully."), CClientUIInterface::MSG_INFORMATION | CClientUIInterface::MODAL); + } else { + Q_EMIT message(tr("Error"), QString::fromStdString(err_string), CClientUIInterface::MSG_ERROR); + } + } else { + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); + Q_EMIT message(tr("PSBT copied"), "Copied to clipboard", CClientUIInterface::MSG_INFORMATION); + return; + } + } + case QMessageBox::Cancel: + break; + default: + assert(false); + } +} + bool WalletView::handlePaymentRequest(const SendCoinsRecipient& recipient) { return sendCoinsPage->handlePaymentRequest(recipient); diff --git a/src/qt/walletview.h b/src/qt/walletview.h index 78d870f59f..11f894e7f8 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -83,6 +83,8 @@ public Q_SLOTS: void gotoSignMessageTab(QString addr = ""); /** Show Sign/Verify Message dialog and switch to verify message tab */ void gotoVerifyMessageTab(QString addr = ""); + /** Load Partially Signed Bitcoin Transaction */ + void gotoLoadPSBT(); /** Show incoming transaction notification for new transactions. diff --git a/src/random.cpp b/src/random.cpp index 2a27e6ba0d..765239e1f5 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/random.h b/src/random.h index 4e4597cff6..0c5008685b 100644 --- a/src/random.h +++ b/src/random.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 8b3d478529..073d82b491 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f71ed99652..827b83d67e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -375,8 +375,6 @@ static UniValue getdifficulty(const JSONRPCRequest& request) static std::vector<RPCResult> MempoolEntryDescription() { return { RPCResult{RPCResult::Type::NUM, "vsize", "virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted."}, - RPCResult{RPCResult::Type::NUM, "size", "(DEPRECATED) same as vsize. Only returned if bitcoind is started with -deprecatedrpc=size\n" - "size will be completely removed in v0.20."}, RPCResult{RPCResult::Type::NUM, "weight", "transaction weight as defined in BIP 141."}, RPCResult{RPCResult::Type::STR_AMOUNT, "fee", "transaction fee in " + CURRENCY_UNIT + " (DEPRECATED)"}, RPCResult{RPCResult::Type::STR_AMOUNT, "modifiedfee", "transaction fee with fee deltas used for mining priority (DEPRECATED)"}, @@ -415,7 +413,6 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("fees", fees); info.pushKV("vsize", (int)e.GetTxSize()); - if (IsDeprecatedRPCEnabled("size")) info.pushKV("size", (int)e.GetTxSize()); info.pushKV("weight", (int)e.GetTxWeight()); info.pushKV("fee", ValueFromAmount(e.GetFee())); info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); @@ -812,7 +809,7 @@ static UniValue getblock(const JSONRPCRequest& request) "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, - {"verbosity", RPCArg::Type::NUM, /* default */ "1", "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, + {"verbosity|verbose", RPCArg::Type::NUM, /* default */ "1", "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, }, { RPCResult{"for verbosity = 0", @@ -1078,13 +1075,11 @@ UniValue gettxout(const JSONRPCRequest& request) static UniValue verifychain(const JSONRPCRequest& request) { - int nCheckLevel = gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL); - int nCheckDepth = gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS); RPCHelpMan{"verifychain", "\nVerifies blockchain database.\n", { - {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", nCheckLevel), "How thorough the block verification is."}, - {"nblocks", RPCArg::Type::NUM, /* default */ strprintf("%d, 0=all", nCheckDepth), "The number of blocks to check."}, + {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", DEFAULT_CHECKLEVEL), "How thorough the block verification is."}, + {"nblocks", RPCArg::Type::NUM, /* default */ strprintf("%d, 0=all", DEFAULT_CHECKBLOCKS), "The number of blocks to check."}, }, RPCResult{ RPCResult::Type::BOOL, "", "Verified or not"}, @@ -1094,15 +1089,12 @@ static UniValue verifychain(const JSONRPCRequest& request) }, }.Check(request); - LOCK(cs_main); + const int check_level(request.params[0].isNull() ? DEFAULT_CHECKLEVEL : request.params[0].get_int()); + const int check_depth{request.params[1].isNull() ? DEFAULT_CHECKBLOCKS : request.params[1].get_int()}; - if (!request.params[0].isNull()) - nCheckLevel = request.params[0].get_int(); - if (!request.params[1].isNull()) - nCheckDepth = request.params[1].get_int(); + LOCK(cs_main); - return CVerifyDB().VerifyDB( - Params(), &::ChainstateActive().CoinsTip(), nCheckLevel, nCheckDepth); + return CVerifyDB().VerifyDB(Params(), &::ChainstateActive().CoinsTip(), check_level, check_depth); } static void BuriedForkDescPushBack(UniValue& softforks, const std::string &name, int height) EXCLUSIVE_LOCKS_REQUIRED(cs_main) @@ -1319,7 +1311,7 @@ static UniValue getchaintips(const JSONRPCRequest& request) /* * Idea: the set of chain tips is ::ChainActive().tip, plus orphan blocks which do not have another orphan building off of them. * Algorithm: - * - Make one pass through g_blockman.m_block_index, picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. + * - Make one pass through BlockIndex(), picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. * - add ::ChainActive().Tip() */ @@ -1726,8 +1718,10 @@ static UniValue getblockstats(const JSONRPCRequest& request) {RPCResult::Type::NUM, "utxo_size_inc", "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, }}, RPCExamples{ - HelpExampleCli("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") - + HelpExampleRpc("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") + HelpExampleCli("getblockstats", R"('"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"' '["minfeerate","avgfeerate"]')") + + HelpExampleCli("getblockstats", R"(1000 '["minfeerate","avgfeerate"]')") + + HelpExampleRpc("getblockstats", R"("00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09", ["minfeerate","avgfeerate"])") + + HelpExampleRpc("getblockstats", R"(1000, ["minfeerate","avgfeerate"])") }, }.Check(request); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index c1762483e9..3045a74d7a 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -33,6 +33,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "generatetoaddress", 2, "maxtries" }, { "generatetodescriptor", 0, "num_blocks" }, { "generatetodescriptor", 2, "maxtries" }, + { "generateblock", 1, "transactions" }, { "getnetworkhashps", 0, "nblocks" }, { "getnetworkhashps", 1, "height" }, { "sendtoaddress", 1, "amount" }, @@ -97,10 +98,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "signrawtransactionwithkey", 1, "privkeys" }, { "signrawtransactionwithkey", 2, "prevtxs" }, { "signrawtransactionwithwallet", 1, "prevtxs" }, - { "sendrawtransaction", 1, "allowhighfees" }, { "sendrawtransaction", 1, "maxfeerate" }, { "testmempoolaccept", 0, "rawtxs" }, - { "testmempoolaccept", 1, "allowhighfees" }, { "testmempoolaccept", 1, "maxfeerate" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, @@ -132,6 +131,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "importpubkey", 2, "rescan" }, { "importmulti", 0, "requests" }, { "importmulti", 1, "options" }, + { "importdescriptors", 0, "requests" }, { "verifychain", 0, "checklevel" }, { "verifychain", 1, "nblocks" }, { "getblockstats", 0, "hash_or_height" }, @@ -154,6 +154,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "logging", 0, "include" }, { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, + { "upgradewallet", 0, "version" }, // Echo with conversion (For testing only) { "echojson", 0, "arg0" }, { "echojson", 1, "arg1" }, @@ -170,6 +171,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createwallet", 1, "disable_private_keys"}, { "createwallet", 2, "blank"}, { "createwallet", 4, "avoid_reuse"}, + { "createwallet", 5, "descriptors"}, { "getnodeaddresses", 0, "count"}, { "stop", 0, "wait" }, }; diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index da9d583fa7..05d3fd6afb 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -101,6 +101,36 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].get_int() : 120, !request.params[1].isNull() ? request.params[1].get_int() : -1); } +static bool GenerateBlock(CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) +{ + block_hash.SetNull(); + + { + LOCK(cs_main); + IncrementExtraNonce(&block, ::ChainActive().Tip(), extra_nonce); + } + + CChainParams chainparams(Params()); + + while (max_tries > 0 && block.nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus()) && !ShutdownRequested()) { + ++block.nNonce; + --max_tries; + } + if (max_tries == 0 || ShutdownRequested()) { + return false; + } + if (block.nNonce == std::numeric_limits<uint32_t>::max()) { + return true; + } + + std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); + if (!ProcessNewBlock(chainparams, shared_pblock, true, nullptr)) + throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); + + block_hash = block.GetHash(); + return true; +} + static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { int nHeightEnd = 0; @@ -119,29 +149,54 @@ static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbas if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; - { - LOCK(cs_main); - IncrementExtraNonce(pblock, ::ChainActive().Tip(), nExtraNonce); - } - while (nMaxTries > 0 && pblock->nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(pblock->GetHash(), pblock->nBits, Params().GetConsensus()) && !ShutdownRequested()) { - ++pblock->nNonce; - --nMaxTries; - } - if (nMaxTries == 0 || ShutdownRequested()) { + + uint256 block_hash; + if (!GenerateBlock(*pblock, nMaxTries, nExtraNonce, block_hash)) { break; } - if (pblock->nNonce == std::numeric_limits<uint32_t>::max()) { - continue; + + if (!block_hash.IsNull()) { + ++nHeight; + blockHashes.push_back(block_hash.GetHex()); } - std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - if (!ProcessNewBlock(Params(), shared_pblock, true, nullptr)) - throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); - ++nHeight; - blockHashes.push_back(pblock->GetHash().GetHex()); } return blockHashes; } +static bool getScriptFromDescriptor(const std::string& descriptor, CScript& script, std::string& error) +{ + FlatSigningProvider key_provider; + const auto desc = Parse(descriptor, key_provider, error, /* require_checksum = */ false); + if (desc) { + if (desc->IsRange()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptor not accepted. Maybe pass through deriveaddresses first?"); + } + + FlatSigningProvider provider; + std::vector<CScript> scripts; + if (!desc->Expand(0, key_provider, scripts, provider)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); + } + + // Combo desriptors can have 2 or 4 scripts, so we can't just check scripts.size() == 1 + CHECK_NONFATAL(scripts.size() > 0 && scripts.size() <= 4); + + if (scripts.size() == 1) { + script = scripts.at(0); + } else if (scripts.size() == 4) { + // For uncompressed keys, take the 3rd script, since it is p2wpkh + script = scripts.at(2); + } else { + // Else take the 2nd script, since it is p2pkh + script = scripts.at(1); + } + + return true; + } else { + return false; + } +} + static UniValue generatetodescriptor(const JSONRPCRequest& request) { RPCHelpMan{ @@ -166,27 +221,15 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) const int num_blocks{request.params[0].get_int()}; const int64_t max_tries{request.params[2].isNull() ? 1000000 : request.params[2].get_int()}; - FlatSigningProvider key_provider; + CScript coinbase_script; std::string error; - const auto desc = Parse(request.params[1].get_str(), key_provider, error, /* require_checksum = */ false); - if (!desc) { + if (!getScriptFromDescriptor(request.params[1].get_str(), coinbase_script, error)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); } - if (desc->IsRange()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptor not accepted. Maybe pass through deriveaddresses first?"); - } - - FlatSigningProvider provider; - std::vector<CScript> coinbase_script; - if (!desc->Expand(0, key_provider, coinbase_script, provider)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); - } const CTxMemPool& mempool = EnsureMemPool(); - CHECK_NONFATAL(coinbase_script.size() == 1); - - return generateBlocks(mempool, coinbase_script.at(0), num_blocks, max_tries); + return generateBlocks(mempool, coinbase_script, num_blocks, max_tries); } static UniValue generatetoaddress(const JSONRPCRequest& request) @@ -229,6 +272,113 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) return generateBlocks(mempool, coinbase_script, nGenerate, nMaxTries); } +static UniValue generateblock(const JSONRPCRequest& request) +{ + RPCHelpMan{"generateblock", + "\nMine a block with a set of ordered transactions immediately to a specified address or descriptor (before the RPC call returns)\n", + { + {"output", RPCArg::Type::STR, RPCArg::Optional::NO, "The address or descriptor to send the newly generated bitcoin to."}, + {"transactions", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings which are either txids or raw transactions.\n" + "Txids must reference transactions currently in the mempool.\n" + "All transactions must be valid and in valid order, otherwise the block will be rejected.", + { + {"rawtx/txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, + }, + }, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "hash", "hash of generated block"}, + } + }, + RPCExamples{ + "\nGenerate a block to myaddress, with txs rawtx and mempool_txid\n" + + HelpExampleCli("generateblock", R"("myaddress" '["rawtx", "mempool_txid"]')") + }, + }.Check(request); + + const auto address_or_descriptor = request.params[0].get_str(); + CScript coinbase_script; + std::string error; + + if (!getScriptFromDescriptor(address_or_descriptor, coinbase_script, error)) { + const auto destination = DecodeDestination(address_or_descriptor); + if (!IsValidDestination(destination)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address or descriptor"); + } + + coinbase_script = GetScriptForDestination(destination); + } + + const CTxMemPool& mempool = EnsureMemPool(); + + std::vector<CTransactionRef> txs; + const auto raw_txs_or_txids = request.params[1].get_array(); + for (size_t i = 0; i < raw_txs_or_txids.size(); i++) { + const auto str(raw_txs_or_txids[i].get_str()); + + uint256 hash; + CMutableTransaction mtx; + if (ParseHashStr(str, hash)) { + + const auto tx = mempool.get(hash); + if (!tx) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Transaction %s not in mempool.", str)); + } + + txs.emplace_back(tx); + + } else if (DecodeHexTx(mtx, str)) { + txs.push_back(MakeTransactionRef(std::move(mtx))); + + } else { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("Transaction decode failed for %s", str)); + } + } + + CChainParams chainparams(Params()); + CBlock block; + + { + LOCK(cs_main); + + CTxMemPool empty_mempool; + std::unique_ptr<CBlockTemplate> blocktemplate(BlockAssembler(empty_mempool, chainparams).CreateNewBlock(coinbase_script)); + if (!blocktemplate) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); + } + block = blocktemplate->block; + } + + CHECK_NONFATAL(block.vtx.size() == 1); + + // Add transactions + block.vtx.insert(block.vtx.end(), txs.begin(), txs.end()); + RegenerateCommitments(block); + + { + LOCK(cs_main); + + BlockValidationState state; + if (!TestBlockValidity(state, chainparams, block, LookupBlockIndex(block.hashPrevBlock), false, false)) { + throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("TestBlockValidity failed: %s", state.ToString())); + } + } + + uint256 block_hash; + uint64_t max_tries{1000000}; + unsigned int extra_nonce{0}; + + if (!GenerateBlock(block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { + throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block."); + } + + UniValue obj(UniValue::VOBJ); + obj.pushKV("hash", block_hash.GetHex()); + return obj; +} + static UniValue getmininginfo(const JSONRPCRequest& request) { RPCHelpMan{"getmininginfo", @@ -1038,6 +1188,7 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, {"nblocks","address","maxtries"} }, { "generating", "generatetodescriptor", &generatetodescriptor, {"num_blocks","descriptor","maxtries"} }, + { "generating", "generateblock", &generateblock, {"output","transactions"} }, { "util", "estimatesmartfee", &estimatesmartfee, {"conf_target", "estimate_mode"} }, diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 51a9581349..f3c5fed858 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -1,9 +1,10 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <httpserver.h> +#include <interfaces/chain.h> #include <key_io.h> #include <node/context.h> #include <outputtype.h> @@ -363,7 +364,13 @@ static UniValue setmocktime(const JSONRPCRequest& request) LOCK(cs_main); RPCTypeCheck(request.params, {UniValue::VNUM}); - SetMockTime(request.params[0].get_int64()); + int64_t time = request.params[0].get_int64(); + SetMockTime(time); + if (g_rpc_node) { + for (const auto& chain_client : g_rpc_node->chain_clients) { + chain_client->setMockTime(time); + } + } return NullUniValue; } @@ -509,7 +516,7 @@ UniValue logging(const JSONRPCRequest& request) "When called with arguments, adds or removes categories from debug logging and return the lists above.\n" "The arguments are evaluated in order \"include\", \"exclude\".\n" "If an item is both included and excluded, it will thus end up being excluded.\n" - "The valid logging categories are: " + ListLogCategories() + "\n" + "The valid logging categories are: " + LogInstance().LogCategoriesString() + "\n" "In addition, the following are available as category names with special meanings:\n" " - \"all\", \"1\" : represent all logging categories.\n" " - \"none\", \"0\" : even if other logging categories are specified, ignore all of them.\n" @@ -561,8 +568,7 @@ UniValue logging(const JSONRPCRequest& request) } UniValue result(UniValue::VOBJ); - std::vector<CLogCategoryActive> vLogCatActive = ListActiveLogCategories(); - for (const auto& logCatActive : vLogCatActive) { + for (const auto& logCatActive : LogInstance().LogCategoriesList()) { result.pushKV(logCatActive.category, logCatActive.active); } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 10562126db..d6d15f8b56 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 3c0b060439..063ee1697c 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -40,12 +40,6 @@ #include <univalue.h> -/** Maximum fee rate for sendrawtransaction and testmempoolaccept. - * By default, a transaction with a fee rate higher than this will be rejected - * by the RPCs. This can be overridden with the maxfeerate argument. - */ -static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; - static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { // Call into TxToUniv() in bitcoin-common to decode the transaction hex. @@ -825,7 +819,7 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) RPCTypeCheck(request.params, { UniValue::VSTR, - UniValueType(), // NUM or BOOL, checked later + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() }); // parse hex string from parameter @@ -834,13 +828,9 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - CFeeRate max_raw_tx_fee_rate = DEFAULT_MAX_RAW_TX_FEE_RATE; - // TODO: temporary migration code for old clients. Remove in v0.20 - if (request.params[1].isBool()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); - } else if (!request.params[1].isNull()) { - max_raw_tx_fee_rate = CFeeRate(AmountFromValue(request.params[1])); - } + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); int64_t virtual_size = GetVirtualTransactionSize(*tx); CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); @@ -896,7 +886,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) RPCTypeCheck(request.params, { UniValue::VARR, - UniValueType(), // NUM or BOOL, checked later + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() }); if (request.params[0].get_array().size() != 1) { @@ -910,13 +900,9 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) CTransactionRef tx(MakeTransactionRef(std::move(mtx))); const uint256& tx_hash = tx->GetHash(); - CFeeRate max_raw_tx_fee_rate = DEFAULT_MAX_RAW_TX_FEE_RATE; - // TODO: temporary migration code for old clients. Remove in v0.20 - if (request.params[1].isBool()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); - } else if (!request.params[1].isNull()) { - max_raw_tx_fee_rate = CFeeRate(AmountFromValue(request.params[1])); - } + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); CTxMemPool& mempool = EnsureMemPool(); int64_t virtual_size = GetVirtualTransactionSize(*tx); @@ -1823,10 +1809,10 @@ static const CRPCCommand commands[] = { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, - { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","allowhighfees|maxfeerate"} }, + { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","maxfeerate"} }, { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, { "rawtransactions", "signrawtransactionwithkey", &signrawtransactionwithkey, {"hexstring","privkeys","prevtxs","sighashtype"} }, - { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","allowhighfees|maxfeerate"} }, + { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","maxfeerate"} }, { "rawtransactions", "decodepsbt", &decodepsbt, {"psbt"} }, { "rawtransactions", "combinepsbt", &combinepsbt, {"txs"} }, { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index 7b701a2bbe..bd82773bf2 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 436db5dc60..942314eccf 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 7e1fb7a59d..860fa198d5 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,6 +13,9 @@ #include <tuple> +#include <boost/algorithm/string/classification.hpp> +#include <boost/algorithm/string/split.hpp> + const std::string UNIX_EPOCH_TIME = "UNIX epoch time"; const std::string EXAMPLE_ADDRESS[2] = {"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl", "bc1q02ad21edsxd23d32dfgqqsz4vv4nmtfzuklhy3"}; @@ -330,7 +333,7 @@ struct Sections { if (outer_type == OuterType::NONE) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; if (arg.m_type_str.size() != 0 && push_name) { - left += "\"" + arg.m_name + "\": " + arg.m_type_str.at(0); + left += "\"" + arg.GetName() + "\": " + arg.m_type_str.at(0); } else { left += push_name ? arg.ToStringObj(/* oneline */ false) : arg.ToString(/* oneline */ false); } @@ -341,7 +344,7 @@ struct Sections { case RPCArg::Type::OBJ: case RPCArg::Type::OBJ_USER_KEYS: { const auto right = outer_type == OuterType::NONE ? "" : arg.ToDescriptionString(); - PushSection({indent + (push_name ? "\"" + arg.m_name + "\": " : "") + "{", right}); + PushSection({indent + (push_name ? "\"" + arg.GetName() + "\": " : "") + "{", right}); for (const auto& arg_inner : arg.m_inner) { Push(arg_inner, current_indent + 2, OuterType::OBJ); } @@ -353,7 +356,7 @@ struct Sections { } case RPCArg::Type::ARR: { auto left = indent; - left += push_name ? "\"" + arg.m_name + "\": " : ""; + left += push_name ? "\"" + arg.GetName() + "\": " : ""; left += "["; const auto right = outer_type == OuterType::NONE ? "" : arg.ToDescriptionString(); PushSection({left, right}); @@ -419,8 +422,12 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP { std::set<std::string> named_args; for (const auto& arg : m_args) { + std::vector<std::string> names; + boost::split(names, arg.m_names, boost::is_any_of("|")); // Should have unique named arguments - CHECK_NONFATAL(named_args.insert(arg.m_name).second); + for (const std::string& name : names) { + CHECK_NONFATAL(named_args.insert(name).second); + } } } @@ -489,7 +496,7 @@ std::string RPCHelpMan::ToString() const if (i == 0) ret += "\nArguments:\n"; // Push named argument name and description - sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.m_name, arg.ToDescriptionString()); + sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString()); sections.m_max_pad = std::max(sections.m_max_pad, sections.m_sections.back().m_left.size()); // Recursively push nested args @@ -506,6 +513,17 @@ std::string RPCHelpMan::ToString() const return ret; } +std::string RPCArg::GetFirstName() const +{ + return m_names.substr(0, m_names.find("|")); +} + +std::string RPCArg::GetName() const +{ + CHECK_NONFATAL(std::string::npos == m_names.find("|")); + return m_names; +} + bool RPCArg::IsOptional() const { if (m_fallback.which() == 1) { @@ -681,7 +699,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const { std::string res; res += "\""; - res += m_name; + res += GetFirstName(); if (oneline) { res += "\":"; } else { @@ -723,13 +741,13 @@ std::string RPCArg::ToString(const bool oneline) const switch (m_type) { case Type::STR_HEX: case Type::STR: { - return "\"" + m_name + "\""; + return "\"" + GetFirstName() + "\""; } case Type::NUM: case Type::RANGE: case Type::AMOUNT: case Type::BOOL: { - return m_name; + return GetFirstName(); } case Type::OBJ: case Type::OBJ_USER_KEYS: { diff --git a/src/rpc/util.h b/src/rpc/util.h index f65ad1246b..53dce2c397 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -142,7 +142,7 @@ struct RPCArg { OMITTED, }; using Fallback = boost::variant<Optional, /* default value for optional args */ std::string>; - const std::string m_name; //!< The name of the arg (can be empty for inner args) + const std::string m_names; //!< The name of the arg (can be empty for inner args, can contain multiple aliases separated by | for named request arguments) const Type m_type; const std::vector<RPCArg> m_inner; //!< Only used for arrays or dicts const Fallback m_fallback; @@ -157,7 +157,7 @@ struct RPCArg { const std::string description, const std::string oneline_description = "", const std::vector<std::string> type_str = {}) - : m_name{std::move(name)}, + : m_names{std::move(name)}, m_type{std::move(type)}, m_fallback{std::move(fallback)}, m_description{std::move(description)}, @@ -175,7 +175,7 @@ struct RPCArg { const std::vector<RPCArg> inner, const std::string oneline_description = "", const std::vector<std::string> type_str = {}) - : m_name{std::move(name)}, + : m_names{std::move(name)}, m_type{std::move(type)}, m_inner{std::move(inner)}, m_fallback{std::move(fallback)}, @@ -188,6 +188,12 @@ struct RPCArg { bool IsOptional() const; + /** Return the first of all aliases */ + std::string GetFirstName() const; + + /** Return the name, throws when there are aliases */ + std::string GetName() const; + /** * Return the type string of the argument. * Set oneline to allow it to be overridden by a custom oneline type string (m_oneline_description). diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 4cac5a54e0..c4bd47310b 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 83dc046ca1..5279f40506 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -575,6 +575,7 @@ public: default: return nullopt; } } + bool IsSingleType() const final { return true; } }; /** A parsed raw(H) descriptor. */ @@ -602,6 +603,7 @@ public: default: return nullopt; } } + bool IsSingleType() const final { return true; } }; /** A parsed pk(P) descriptor. */ @@ -611,6 +613,7 @@ protected: std::vector<CScript> MakeScripts(const std::vector<CPubKey>& keys, const CScript*, FlatSigningProvider&) const override { return Vector(GetScriptForRawPubKey(keys[0])); } public: PKDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "pk") {} + bool IsSingleType() const final { return true; } }; /** A parsed pkh(P) descriptor. */ @@ -626,6 +629,7 @@ protected: public: PKHDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "pkh") {} Optional<OutputType> GetOutputType() const override { return OutputType::LEGACY; } + bool IsSingleType() const final { return true; } }; /** A parsed wpkh(P) descriptor. */ @@ -641,6 +645,7 @@ protected: public: WPKHDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "wpkh") {} Optional<OutputType> GetOutputType() const override { return OutputType::BECH32; } + bool IsSingleType() const final { return true; } }; /** A parsed combo(P) descriptor. */ @@ -664,6 +669,7 @@ protected: } public: ComboDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "combo") {} + bool IsSingleType() const final { return false; } }; /** A parsed multi(...) or sortedmulti(...) descriptor */ @@ -683,6 +689,7 @@ protected: } public: MultisigDescriptor(int threshold, std::vector<std::unique_ptr<PubkeyProvider>> providers, bool sorted = false) : DescriptorImpl(std::move(providers), {}, sorted ? "sortedmulti" : "multi"), m_threshold(threshold), m_sorted(sorted) {} + bool IsSingleType() const final { return true; } }; /** A parsed sh(...) descriptor. */ @@ -699,6 +706,7 @@ public: if (m_subdescriptor_arg->GetOutputType() == OutputType::BECH32) return OutputType::P2SH_SEGWIT; return OutputType::LEGACY; } + bool IsSingleType() const final { return true; } }; /** A parsed wsh(...) descriptor. */ @@ -709,6 +717,7 @@ protected: public: WSHDescriptor(std::unique_ptr<DescriptorImpl> desc) : DescriptorImpl({}, std::move(desc), "wsh") {} Optional<OutputType> GetOutputType() const override { return OutputType::BECH32; } + bool IsSingleType() const final { return true; } }; //////////////////////////////////////////////////////////////////////////// diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 34cd5760de..17b43e7c81 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -87,6 +87,9 @@ struct Descriptor { /** Convert the descriptor back to a string, undoing parsing. */ virtual std::string ToString() const = 0; + /** Whether this descriptor will return one scriptPubKey or multiple (aka is or is not combo) */ + virtual bool IsSingleType() const = 0; + /** Convert the descriptor to a private string. This fails if the provided provider does not have the relevant private keys. */ virtual bool ToPrivateString(const SigningProvider& provider, std::string& out) const = 0; diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 083022fbdd..23d5b72a5c 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -342,6 +342,35 @@ public: }; } +/** Helper for OP_CHECKSIG and OP_CHECKSIGVERIFY + * + * A return value of false means the script fails entirely. When true is returned, the + * fSuccess variable indicates whether the signature check itself succeeded. + */ +static bool EvalChecksig(const valtype& vchSig, const valtype& vchPubKey, CScript::const_iterator pbegincodehash, CScript::const_iterator pend, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* serror, bool& fSuccess) +{ + // Subset of script starting at the most recent codeseparator + CScript scriptCode(pbegincodehash, pend); + + // Drop the signature in pre-segwit scripts but not segwit scripts + if (sigversion == SigVersion::BASE) { + int found = FindAndDelete(scriptCode, CScript() << vchSig); + if (found > 0 && (flags & SCRIPT_VERIFY_CONST_SCRIPTCODE)) + return set_error(serror, SCRIPT_ERR_SIG_FINDANDDELETE); + } + + if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, sigversion, serror)) { + //serror is set + return false; + } + fSuccess = checker.CheckSig(vchSig, vchPubKey, scriptCode, sigversion); + + if (!fSuccess && (flags & SCRIPT_VERIFY_NULLFAIL) && vchSig.size()) + return set_error(serror, SCRIPT_ERR_SIG_NULLFAIL); + + return true; +} + bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& script, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* serror) { static const CScriptNum bnZero(0); @@ -985,25 +1014,8 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& valtype& vchSig = stacktop(-2); valtype& vchPubKey = stacktop(-1); - // Subset of script starting at the most recent codeseparator - CScript scriptCode(pbegincodehash, pend); - - // Drop the signature in pre-segwit scripts but not segwit scripts - if (sigversion == SigVersion::BASE) { - int found = FindAndDelete(scriptCode, CScript() << vchSig); - if (found > 0 && (flags & SCRIPT_VERIFY_CONST_SCRIPTCODE)) - return set_error(serror, SCRIPT_ERR_SIG_FINDANDDELETE); - } - - if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, sigversion, serror)) { - //serror is set - return false; - } - bool fSuccess = checker.CheckSig(vchSig, vchPubKey, scriptCode, sigversion); - - if (!fSuccess && (flags & SCRIPT_VERIFY_NULLFAIL) && vchSig.size()) - return set_error(serror, SCRIPT_ERR_SIG_NULLFAIL); - + bool fSuccess = true; + if (!EvalChecksig(vchSig, vchPubKey, pbegincodehash, pend, flags, checker, sigversion, serror, fSuccess)) return false; popstack(stack); popstack(stack); stack.push_back(fSuccess ? vchTrue : vchFalse); @@ -1279,18 +1291,29 @@ uint256 GetOutputsHash(const T& txTo) } // namespace template <class T> -PrecomputedTransactionData::PrecomputedTransactionData(const T& txTo) +void PrecomputedTransactionData::Init(const T& txTo) { + assert(!m_ready); + // Cache is calculated only for transactions with witness if (txTo.HasWitness()) { hashPrevouts = GetPrevoutHash(txTo); hashSequence = GetSequenceHash(txTo); hashOutputs = GetOutputsHash(txTo); - ready = true; } + + m_ready = true; +} + +template <class T> +PrecomputedTransactionData::PrecomputedTransactionData(const T& txTo) +{ + Init(txTo); } // explicit instantiation +template void PrecomputedTransactionData::Init(const CTransaction& txTo); +template void PrecomputedTransactionData::Init(const CMutableTransaction& txTo); template PrecomputedTransactionData::PrecomputedTransactionData(const CTransaction& txTo); template PrecomputedTransactionData::PrecomputedTransactionData(const CMutableTransaction& txTo); @@ -1303,7 +1326,7 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn uint256 hashPrevouts; uint256 hashSequence; uint256 hashOutputs; - const bool cacheready = cache && cache->ready; + const bool cacheready = cache && cache->m_ready; if (!(nHashType & SIGHASH_ANYONECANPAY)) { hashPrevouts = cacheready ? cache->hashPrevouts : GetPrevoutHash(txTo); diff --git a/src/script/interpreter.h b/src/script/interpreter.h index 2b104a608c..71f2436369 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -121,7 +121,12 @@ bool CheckSignatureEncoding(const std::vector<unsigned char> &vchSig, unsigned i struct PrecomputedTransactionData { uint256 hashPrevouts, hashSequence, hashOutputs; - bool ready = false; + bool m_ready = false; + + PrecomputedTransactionData() = default; + + template <class T> + void Init(const T& tx); template <class T> explicit PrecomputedTransactionData(const T& tx); diff --git a/src/script/script.h b/src/script/script.h index 7aaa10b60b..daf4224530 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -419,28 +419,15 @@ public: READWRITEAS(CScriptBase, *this); } - CScript& operator+=(const CScript& b) - { - reserve(size() + b.size()); - insert(end(), b.begin(), b.end()); - return *this; - } - - friend CScript operator+(const CScript& a, const CScript& b) - { - CScript ret = a; - ret += b; - return ret; - } - - CScript(int64_t b) { operator<<(b); } - + explicit CScript(int64_t b) { operator<<(b); } explicit CScript(opcodetype b) { operator<<(b); } explicit CScript(const CScriptNum& b) { operator<<(b); } // delete non-existent constructor to defend against future introduction // e.g. via prevector explicit CScript(const std::vector<unsigned char>& b) = delete; + /** Delete non-existent operator to defend against future introduction */ + CScript& operator<<(const CScript& b) = delete; CScript& operator<<(int64_t b) { return push_int64(b); } @@ -487,15 +474,6 @@ public: return *this; } - CScript& operator<<(const CScript& b) - { - // I'm not sure if this should push the script or concatenate scripts. - // If there's ever a use for pushing a script onto a script, delete this member fn - assert(!"Warning: Pushing a CScript onto a CScript with << is probably not intended, use + to concatenate!"); - return *this; - } - - bool GetOp(const_iterator& pc, opcodetype& opcodeRet, std::vector<unsigned char>& vchRet) const { return GetScriptOp(pc, end(), opcodeRet, &vchRet); @@ -506,7 +484,6 @@ public: return GetScriptOp(pc, end(), opcodeRet, nullptr); } - /** Encode/decode small integers: */ static int DecodeOP_N(opcodetype opcode) { diff --git a/src/script/script_error.cpp b/src/script/script_error.cpp index ff521d5860..57e8fee539 100644 --- a/src/script/script_error.cpp +++ b/src/script/script_error.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sign.cpp b/src/script/sign.cpp index fe8292fe57..1e00afcf89 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sign.h b/src/script/sign.h index f03af0713f..b77d26c0d7 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/serialize.h b/src/serialize.h index 5045cb3c7f..fe53eeed31 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/span.h b/src/span.h index 9379b15c81..664ea8d4c6 100644 --- a/src/span.h +++ b/src/span.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index f3cc12201c..9b14637b95 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/sync.h b/src/sync.h index ead2cdc67b..0c6f0ef0a7 100644 --- a/src/sync.h +++ b/src/sync.h @@ -210,7 +210,7 @@ public: friend class reverse_lock; }; -#define REVERSE_LOCK(g) decltype(g)::reverse_lock PASTE2(revlock, __COUNTER__)(g, #g, __FILE__, __LINE__) +#define REVERSE_LOCK(g) typename std::decay<decltype(g)>::type::reverse_lock PASTE2(revlock, __COUNTER__)(g, #g, __FILE__, __LINE__) template<typename MutexArg> using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove_pointer<MutexArg>::type>::type>; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index dfa8a6df21..bc6b38c682 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -1,18 +1,19 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> +#include <test/data/asmap.raw.h> #include <test/util/setup_common.h> -#include <string> -#include <boost/test/unit_test.hpp> #include <util/asmap.h> #include <util/string.h> -#include <test/data/asmap.raw.h> - #include <hash.h> #include <netbase.h> #include <random.h> +#include <boost/test/unit_test.hpp> + +#include <string> + class CAddrManTest : public CAddrMan { private: diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index 3723a48903..a135c93786 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -1,17 +1,19 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <arith_uint256.h> +#include <test/util/setup_common.h> +#include <uint256.h> + #include <boost/test/unit_test.hpp> -#include <stdint.h> -#include <sstream> + +#include <cmath> #include <iomanip> #include <limits> -#include <cmath> -#include <uint256.h> -#include <arith_uint256.h> +#include <sstream> +#include <stdint.h> #include <string> -#include <test/util/setup_common.h> BOOST_FIXTURE_TEST_SUITE(arith_uint256_tests, BasicTestingSetup) diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index 690368b177..eedab30576 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 94df4d1955..5927eab6cf 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index 53df032252..32329eb510 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2019 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <key.h> #include <key_io.h> #include <streams.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <string> #include <vector> diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index aa704642bf..c8e8cdeeb3 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <chain.h> #include <rpc/blockchain.h> -#include <util/string.h> #include <test/util/setup_common.h> +#include <util/string.h> /* Equality between doubles is imprecise. Comparison should be done * with a small threshold of tolerance, rather than exact equality. diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 5e52dc268f..ff01f730a3 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 4a7ad9b38b..a75d9fc9ec 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,10 +12,10 @@ #include <random.h> #include <serialize.h> #include <streams.h> +#include <test/util/setup_common.h> #include <uint256.h> -#include <util/system.h> #include <util/strencodings.h> -#include <test/util/setup_common.h> +#include <util/system.h> #include <vector> @@ -27,6 +27,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) { CBloomFilter filter(3, 0.01, 0, BLOOM_UPDATE_ALL); + BOOST_CHECK_MESSAGE( !filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter should be empty!"); filter.insert(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")); BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter doesn't contain just-inserted object!"); // One bit different in first byte @@ -50,8 +51,6 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter doesn't contain just-inserted object!"); - filter.clear(); - BOOST_CHECK_MESSAGE( !filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter should be empty!"); } BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) diff --git a/src/test/bswap_tests.cpp b/src/test/bswap_tests.cpp index 0b4bfdb019..c89cb5488d 100644 --- a/src/test/bswap_tests.cpp +++ b/src/test/bswap_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index a9628e85f9..0565982215 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -1,23 +1,23 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <checkqueue.h> +#include <test/util/setup_common.h> #include <util/memory.h> #include <util/system.h> #include <util/time.h> -#include <test/util/setup_common.h> -#include <checkqueue.h> #include <boost/test/unit_test.hpp> #include <boost/thread.hpp> + #include <atomic> -#include <thread> -#include <vector> -#include <mutex> #include <condition_variable> - +#include <mutex> +#include <thread> #include <unordered_set> #include <utility> +#include <vector> BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, TestingSetup) diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 436c1bffa0..60196c36a5 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -8,6 +8,7 @@ #include <script/standard.h> #include <streams.h> #include <test/util/setup_common.h> +#include <txdb.h> #include <uint256.h> #include <undo.h> #include <util/strencodings.h> @@ -109,7 +110,12 @@ static const unsigned int NUM_SIMULATION_ITERATIONS = 40000; // // During the process, booleans are kept to make sure that the randomized // operation hits all branches. -BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) +// +// If fake_best_block is true, assign a random uint256 to mock the recording +// of best block on flush. This is necessary when using CCoinsViewDB as the base, +// otherwise we'll hit an assertion in BatchWrite. +// +void SimulationTest(CCoinsView* base, bool fake_best_block) { // Various coverage trackers. bool removed_all_caches = false; @@ -126,9 +132,8 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) std::map<COutPoint, Coin> result; // The cache stack. - CCoinsViewTest base; // A CCoinsViewTest at the bottom. std::vector<CCoinsViewCacheTest*> stack; // A stack of CCoinsViewCaches on top. - stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. + stack.push_back(new CCoinsViewCacheTest(base)); // Start with one cache. // Use a limited set of random transaction ids, so we do test overwriting entries. std::vector<uint256> txids; @@ -211,6 +216,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Every 100 iterations, flush an intermediate cache if (stack.size() > 1 && InsecureRandBool() == 0) { unsigned int flushIndex = InsecureRandRange(stack.size() - 1); + if (fake_best_block) stack[flushIndex]->SetBestBlock(InsecureRand256()); BOOST_CHECK(stack[flushIndex]->Flush()); } } @@ -218,13 +224,14 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Every 100 iterations, change the cache stack. if (stack.size() > 0 && InsecureRandBool() == 0) { //Remove the top cache + if (fake_best_block) stack.back()->SetBestBlock(InsecureRand256()); BOOST_CHECK(stack.back()->Flush()); delete stack.back(); stack.pop_back(); } if (stack.size() == 0 || (stack.size() < 4 && InsecureRandBool())) { //Add a new cache - CCoinsView* tip = &base; + CCoinsView* tip = base; if (stack.size() > 0) { tip = stack.back(); } else { @@ -256,6 +263,16 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) BOOST_CHECK(uncached_an_entry); } +// Run the above simulation for multiple base types. +BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) +{ + CCoinsViewTest base; + SimulationTest(&base, false); + + CCoinsViewDB db_base{"test", /*nCacheSize*/ 1 << 23, /*fMemory*/ true, /*fWipe*/ false}; + SimulationTest(&db_base, true); +} + // Store of all necessary tx and undo data for next test typedef std::map<COutPoint, std::tuple<CTransaction,CTxUndo,Coin>> UtxoData; UtxoData utxoData; @@ -532,7 +549,7 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) } const static COutPoint OUTPOINT; -const static CAmount PRUNED = -1; +const static CAmount SPENT = -1; const static CAmount ABSENT = -2; const static CAmount FAIL = -3; const static CAmount VALUE1 = 100; @@ -551,7 +568,7 @@ static void SetCoinsValue(CAmount value, Coin& coin) assert(value != ABSENT); coin.Clear(); assert(coin.IsSpent()); - if (value != PRUNED) { + if (value != SPENT) { coin.out.nValue = value; coin.nHeight = 1; assert(!coin.IsSpent()); @@ -581,7 +598,7 @@ void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) flags = NO_ENTRY; } else { if (it->second.coin.IsSpent()) { - value = PRUNED; + value = SPENT; } else { value = it->second.coin.out.nValue; } @@ -634,28 +651,28 @@ BOOST_AUTO_TEST_CASE(ccoins_access) * Value Value Value Flags Flags */ CheckAccessCoin(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoin(ABSENT, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoin(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, SPENT , SPENT , 0 , 0 ); + CheckAccessCoin(ABSENT, SPENT , SPENT , FRESH , FRESH ); + CheckAccessCoin(ABSENT, SPENT , SPENT , DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, SPENT , SPENT , DIRTY|FRESH, DIRTY|FRESH); CheckAccessCoin(ABSENT, VALUE2, VALUE2, 0 , 0 ); CheckAccessCoin(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoin(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoin(PRUNED, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoin(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoin(PRUNED, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoin(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(SPENT , ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckAccessCoin(SPENT , SPENT , SPENT , 0 , 0 ); + CheckAccessCoin(SPENT , SPENT , SPENT , FRESH , FRESH ); + CheckAccessCoin(SPENT , SPENT , SPENT , DIRTY , DIRTY ); + CheckAccessCoin(SPENT , SPENT , SPENT , DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(SPENT , VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(SPENT , VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(SPENT , VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(SPENT , VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); CheckAccessCoin(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); - CheckAccessCoin(VALUE1, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoin(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, SPENT , SPENT , 0 , 0 ); + CheckAccessCoin(VALUE1, SPENT , SPENT , FRESH , FRESH ); + CheckAccessCoin(VALUE1, SPENT , SPENT , DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, SPENT , SPENT , DIRTY|FRESH, DIRTY|FRESH); CheckAccessCoin(VALUE1, VALUE2, VALUE2, 0 , 0 ); CheckAccessCoin(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); @@ -685,31 +702,31 @@ BOOST_AUTO_TEST_CASE(ccoins_spend) * Value Value Value Flags Flags */ CheckSpendCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckSpendCoins(ABSENT, PRUNED, PRUNED, 0 , DIRTY ); - CheckSpendCoins(ABSENT, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckSpendCoins(ABSENT, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(ABSENT, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, SPENT , SPENT , 0 , DIRTY ); + CheckSpendCoins(ABSENT, SPENT , ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, SPENT , SPENT , DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, SPENT , ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, SPENT , 0 , DIRTY ); CheckSpendCoins(ABSENT, VALUE2, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(ABSENT, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, SPENT , DIRTY , DIRTY ); CheckSpendCoins(ABSENT, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckSpendCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckSpendCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckSpendCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(PRUNED, VALUE2, PRUNED, 0 , DIRTY ); - CheckSpendCoins(PRUNED, VALUE2, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(PRUNED, VALUE2, PRUNED, DIRTY , DIRTY ); - CheckSpendCoins(PRUNED, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(VALUE1, ABSENT, PRUNED, NO_ENTRY , DIRTY ); - CheckSpendCoins(VALUE1, PRUNED, PRUNED, 0 , DIRTY ); - CheckSpendCoins(VALUE1, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckSpendCoins(VALUE1, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(VALUE1, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(SPENT , ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(SPENT , SPENT , SPENT , 0 , DIRTY ); + CheckSpendCoins(SPENT , SPENT , ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(SPENT , SPENT , SPENT , DIRTY , DIRTY ); + CheckSpendCoins(SPENT , SPENT , ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(SPENT , VALUE2, SPENT , 0 , DIRTY ); + CheckSpendCoins(SPENT , VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(SPENT , VALUE2, SPENT , DIRTY , DIRTY ); + CheckSpendCoins(SPENT , VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, ABSENT, SPENT , NO_ENTRY , DIRTY ); + CheckSpendCoins(VALUE1, SPENT , SPENT , 0 , DIRTY ); + CheckSpendCoins(VALUE1, SPENT , ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, SPENT , SPENT , DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, SPENT , ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, SPENT , 0 , DIRTY ); CheckSpendCoins(VALUE1, VALUE2, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(VALUE1, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, SPENT , DIRTY , DIRTY ); CheckSpendCoins(VALUE1, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); } @@ -742,7 +759,7 @@ static void CheckAddCoinBase(CAmount base_value, CAmount cache_value, CAmount mo template <typename... Args> static void CheckAddCoin(Args&&... args) { - for (const CAmount base_value : {ABSENT, PRUNED, VALUE1}) + for (const CAmount base_value : {ABSENT, SPENT, VALUE1}) CheckAddCoinBase(base_value, std::forward<Args>(args)...); } @@ -751,21 +768,21 @@ BOOST_AUTO_TEST_CASE(ccoins_add) /* Check AddCoin behavior, requesting a new coin from a cache view, * writing a modification to the coin, and then checking the resulting * entry in the cache after the modification. Verify behavior with the - * with the AddCoin potential_overwrite argument set to false, and to true. + * AddCoin possible_overwrite argument set to false, and to true. * - * Cache Write Result Cache Result potential_overwrite + * Cache Write Result Cache Result possible_overwrite * Value Value Value Flags Flags */ CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); - CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); - CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); - CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); - CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); - CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(SPENT , VALUE3, VALUE3, 0 , DIRTY|FRESH, false); + CheckAddCoin(SPENT , VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(SPENT , VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); + CheckAddCoin(SPENT , VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY , DIRTY , false); + CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); + CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); CheckAddCoin(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); CheckAddCoin(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); CheckAddCoin(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); @@ -805,42 +822,42 @@ BOOST_AUTO_TEST_CASE(ccoins_write) * Value Value Value Flags Flags Flags */ CheckWriteCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY , NO_ENTRY ); - CheckWriteCoins(ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY , DIRTY ); - CheckWriteCoins(ABSENT, PRUNED, ABSENT, NO_ENTRY , DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(ABSENT, SPENT , SPENT , NO_ENTRY , DIRTY , DIRTY ); + CheckWriteCoins(ABSENT, SPENT , ABSENT, NO_ENTRY , DIRTY|FRESH, NO_ENTRY ); CheckWriteCoins(ABSENT, VALUE2, VALUE2, NO_ENTRY , DIRTY , DIRTY ); CheckWriteCoins(ABSENT, VALUE2, VALUE2, NO_ENTRY , DIRTY|FRESH, DIRTY|FRESH); - CheckWriteCoins(PRUNED, ABSENT, PRUNED, 0 , NO_ENTRY , 0 ); - CheckWriteCoins(PRUNED, ABSENT, PRUNED, FRESH , NO_ENTRY , FRESH ); - CheckWriteCoins(PRUNED, ABSENT, PRUNED, DIRTY , NO_ENTRY , DIRTY ); - CheckWriteCoins(PRUNED, ABSENT, PRUNED, DIRTY|FRESH, NO_ENTRY , DIRTY|FRESH); - CheckWriteCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY , DIRTY ); - CheckWriteCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(PRUNED, PRUNED, ABSENT, FRESH , DIRTY , NO_ENTRY ); - CheckWriteCoins(PRUNED, PRUNED, ABSENT, FRESH , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , DIRTY ); - CheckWriteCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, DIRTY , NO_ENTRY ); - CheckWriteCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, 0 , DIRTY , DIRTY ); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, 0 , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, FRESH , DIRTY , DIRTY|FRESH); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, FRESH , DIRTY|FRESH, DIRTY|FRESH); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY , DIRTY ); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY , DIRTY|FRESH); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH, DIRTY|FRESH); + CheckWriteCoins(SPENT , ABSENT, SPENT , 0 , NO_ENTRY , 0 ); + CheckWriteCoins(SPENT , ABSENT, SPENT , FRESH , NO_ENTRY , FRESH ); + CheckWriteCoins(SPENT , ABSENT, SPENT , DIRTY , NO_ENTRY , DIRTY ); + CheckWriteCoins(SPENT , ABSENT, SPENT , DIRTY|FRESH, NO_ENTRY , DIRTY|FRESH); + CheckWriteCoins(SPENT , SPENT , SPENT , 0 , DIRTY , DIRTY ); + CheckWriteCoins(SPENT , SPENT , SPENT , 0 , DIRTY|FRESH, DIRTY ); + CheckWriteCoins(SPENT , SPENT , ABSENT, FRESH , DIRTY , NO_ENTRY ); + CheckWriteCoins(SPENT , SPENT , ABSENT, FRESH , DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(SPENT , SPENT , SPENT , DIRTY , DIRTY , DIRTY ); + CheckWriteCoins(SPENT , SPENT , SPENT , DIRTY , DIRTY|FRESH, DIRTY ); + CheckWriteCoins(SPENT , SPENT , ABSENT, DIRTY|FRESH, DIRTY , NO_ENTRY ); + CheckWriteCoins(SPENT , SPENT , ABSENT, DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(SPENT , VALUE2, VALUE2, 0 , DIRTY , DIRTY ); + CheckWriteCoins(SPENT , VALUE2, VALUE2, 0 , DIRTY|FRESH, DIRTY ); + CheckWriteCoins(SPENT , VALUE2, VALUE2, FRESH , DIRTY , DIRTY|FRESH); + CheckWriteCoins(SPENT , VALUE2, VALUE2, FRESH , DIRTY|FRESH, DIRTY|FRESH); + CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY , DIRTY , DIRTY ); + CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY , DIRTY|FRESH, DIRTY ); + CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY|FRESH, DIRTY , DIRTY|FRESH); + CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH, DIRTY|FRESH); CheckWriteCoins(VALUE1, ABSENT, VALUE1, 0 , NO_ENTRY , 0 ); CheckWriteCoins(VALUE1, ABSENT, VALUE1, FRESH , NO_ENTRY , FRESH ); CheckWriteCoins(VALUE1, ABSENT, VALUE1, DIRTY , NO_ENTRY , DIRTY ); CheckWriteCoins(VALUE1, ABSENT, VALUE1, DIRTY|FRESH, NO_ENTRY , DIRTY|FRESH); - CheckWriteCoins(VALUE1, PRUNED, PRUNED, 0 , DIRTY , DIRTY ); - CheckWriteCoins(VALUE1, PRUNED, FAIL , 0 , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, PRUNED, ABSENT, FRESH , DIRTY , NO_ENTRY ); - CheckWriteCoins(VALUE1, PRUNED, FAIL , FRESH , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY , DIRTY ); - CheckWriteCoins(VALUE1, PRUNED, FAIL , DIRTY , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, PRUNED, ABSENT, DIRTY|FRESH, DIRTY , NO_ENTRY ); - CheckWriteCoins(VALUE1, PRUNED, FAIL , DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , SPENT , 0 , DIRTY , DIRTY ); + CheckWriteCoins(VALUE1, SPENT , FAIL , 0 , DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , ABSENT, FRESH , DIRTY , NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , FAIL , FRESH , DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , SPENT , DIRTY , DIRTY , DIRTY ); + CheckWriteCoins(VALUE1, SPENT , FAIL , DIRTY , DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , ABSENT, DIRTY|FRESH, DIRTY , NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , FAIL , DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); CheckWriteCoins(VALUE1, VALUE2, VALUE2, 0 , DIRTY , DIRTY ); CheckWriteCoins(VALUE1, VALUE2, FAIL , 0 , DIRTY|FRESH, NO_ENTRY ); CheckWriteCoins(VALUE1, VALUE2, VALUE2, FRESH , DIRTY , DIRTY|FRESH); @@ -854,8 +871,8 @@ BOOST_AUTO_TEST_CASE(ccoins_write) // they would be too repetitive (the parent cache is never updated in these // cases). The loop below covers these cases and makes sure the parent cache // is always left unchanged. - for (const CAmount parent_value : {ABSENT, PRUNED, VALUE1}) - for (const CAmount child_value : {ABSENT, PRUNED, VALUE2}) + for (const CAmount parent_value : {ABSENT, SPENT, VALUE1}) + for (const CAmount child_value : {ABSENT, SPENT, VALUE2}) for (const char parent_flags : parent_value == ABSENT ? ABSENT_FLAGS : FLAGS) for (const char child_flags : child_value == ABSENT ? ABSENT_FLAGS : CLEAN_FLAGS) CheckWriteCoins(parent_value, child_value, parent_value, parent_flags, child_flags, parent_flags); diff --git a/src/test/compilerbug_tests.cpp b/src/test/compilerbug_tests.cpp index 1a6fcda009..b68bc279e1 100644 --- a/src/test/compilerbug_tests.cpp +++ b/src/test/compilerbug_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> +#include <test/util/setup_common.h> BOOST_FIXTURE_TEST_SUITE(compilerbug_tests, BasicTestingSetup) diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 22eae91cf0..df1a119d79 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compressor.h> -#include <test/util/setup_common.h> #include <script/standard.h> +#include <test/util/setup_common.h> #include <stdint.h> diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 2deb0c5bfc..f64251fe32 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <crypto/aes.h> #include <crypto/chacha20.h> #include <crypto/chacha_poly_aead.h> -#include <crypto/poly1305.h> #include <crypto/hkdf_sha256_32.h> #include <crypto/hmac_sha256.h> #include <crypto/hmac_sha512.h> +#include <crypto/poly1305.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> #include <crypto/sha512.h> #include <random.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <vector> diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 6be24c0845..3a951d28ae 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> #include <cuckoocache.h> +#include <deque> +#include <random.h> #include <script/sigcache.h> #include <test/util/setup_common.h> -#include <random.h> #include <thread> -#include <deque> /** Test Suite for CuckooCache * diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 3dfae29de6..c378546e8b 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <dbwrapper.h> -#include <uint256.h> #include <test/util/setup_common.h> +#include <uint256.h> #include <util/memory.h> #include <memory> diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 3154c619d2..121b80ab2d 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -1,16 +1,18 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <vector> -#include <string> +#include <script/descriptor.h> #include <script/sign.h> #include <script/standard.h> #include <test/util/setup_common.h> -#include <boost/test/unit_test.hpp> -#include <script/descriptor.h> #include <util/strencodings.h> +#include <boost/test/unit_test.hpp> + +#include <string> +#include <vector> + namespace { void CheckUnparsable(const std::string& prv, const std::string& pub, const std::string& expected_error) diff --git a/src/test/fuzz/addition_overflow.cpp b/src/test/fuzz/addition_overflow.cpp new file mode 100644 index 0000000000..a455992b13 --- /dev/null +++ b/src/test/fuzz/addition_overflow.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +#if defined(__has_builtin) +#if __has_builtin(__builtin_add_overflow) +#define HAVE_BUILTIN_ADD_OVERFLOW +#endif +#elif defined(__GNUC__) && (__GNUC__ >= 5) +#define HAVE_BUILTIN_ADD_OVERFLOW +#endif + +namespace { +template <typename T> +void TestAdditionOverflow(FuzzedDataProvider& fuzzed_data_provider) +{ + const T i = fuzzed_data_provider.ConsumeIntegral<T>(); + const T j = fuzzed_data_provider.ConsumeIntegral<T>(); + const bool is_addition_overflow_custom = AdditionOverflow(i, j); +#if defined(HAVE_BUILTIN_ADD_OVERFLOW) + T result_builtin; + const bool is_addition_overflow_builtin = __builtin_add_overflow(i, j, &result_builtin); + assert(is_addition_overflow_custom == is_addition_overflow_builtin); + if (!is_addition_overflow_custom) { + assert(i + j == result_builtin); + } +#else + if (!is_addition_overflow_custom) { + (void)(i + j); + } +#endif +} +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + TestAdditionOverflow<int64_t>(fuzzed_data_provider); + TestAdditionOverflow<uint64_t>(fuzzed_data_provider); + TestAdditionOverflow<int32_t>(fuzzed_data_provider); + TestAdditionOverflow<uint32_t>(fuzzed_data_provider); + TestAdditionOverflow<int16_t>(fuzzed_data_provider); + TestAdditionOverflow<uint16_t>(fuzzed_data_provider); + TestAdditionOverflow<char>(fuzzed_data_provider); + TestAdditionOverflow<unsigned char>(fuzzed_data_provider); + TestAdditionOverflow<signed char>(fuzzed_data_provider); +} diff --git a/src/test/fuzz/base_encode_decode.cpp b/src/test/fuzz/base_encode_decode.cpp index adad6b3f96..8d49f93c2f 100644 --- a/src/test/fuzz/base_encode_decode.cpp +++ b/src/test/fuzz/base_encode_decode.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,8 +6,8 @@ #include <base58.h> #include <psbt.h> -#include <util/string.h> #include <util/strencodings.h> +#include <util/string.h> #include <cassert> #include <cstdint> diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index 9d0ad369a2..f30fa03e0b 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,8 +7,8 @@ #include <consensus/validation.h> #include <core_io.h> #include <core_memusage.h> -#include <pubkey.h> #include <primitives/block.h> +#include <pubkey.h> #include <streams.h> #include <test/fuzz/fuzz.h> #include <validation.h> @@ -62,4 +62,8 @@ void test_one_input(const std::vector<uint8_t>& buffer) const size_t raw_memory_size = RecursiveDynamicUsage(block); const size_t raw_memory_size_as_shared_ptr = RecursiveDynamicUsage(std::make_shared<CBlock>(block)); assert(raw_memory_size_as_shared_ptr > raw_memory_size); + CBlock block_copy = block; + block_copy.SetNull(); + const bool is_null = block_copy.IsNull(); + assert(is_null); } diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index d1112f8e62..50036ce5bd 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -25,7 +25,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; while (fuzzed_data_provider.remaining_bytes() > 0) { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 6)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 4)) { case 0: { const std::vector<unsigned char> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); (void)bloom_filter.contains(b); @@ -56,13 +56,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(present); break; } - case 3: - bloom_filter.clear(); - break; - case 4: - bloom_filter.reset(fuzzed_data_provider.ConsumeIntegral<unsigned int>()); - break; - case 5: { + case 3: { const Optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mut_tx) { break; @@ -71,7 +65,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)bloom_filter.IsRelevantAndUpdate(tx); break; } - case 6: + case 4: bloom_filter.UpdateEmptyFull(); break; } diff --git a/src/test/fuzz/checkqueue.cpp b/src/test/fuzz/checkqueue.cpp new file mode 100644 index 0000000000..2ed097b827 --- /dev/null +++ b/src/test/fuzz/checkqueue.cpp @@ -0,0 +1,65 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <checkqueue.h> +#include <optional.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +namespace { +struct DumbCheck { + const bool result = false; + + DumbCheck() = default; + + explicit DumbCheck(const bool _result) : result(_result) + { + } + + bool operator()() const + { + return result; + } + + void swap(DumbCheck& x) + { + } +}; +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const unsigned int batch_size = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 1024); + CCheckQueue<DumbCheck> check_queue_1{batch_size}; + CCheckQueue<DumbCheck> check_queue_2{batch_size}; + std::vector<DumbCheck> checks_1; + std::vector<DumbCheck> checks_2; + const int size = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1024); + for (int i = 0; i < size; ++i) { + const bool result = fuzzed_data_provider.ConsumeBool(); + checks_1.emplace_back(result); + checks_2.emplace_back(result); + } + if (fuzzed_data_provider.ConsumeBool()) { + check_queue_1.Add(checks_1); + } + if (fuzzed_data_provider.ConsumeBool()) { + (void)check_queue_1.Wait(); + } + + CCheckQueueControl<DumbCheck> check_queue_control{&check_queue_2}; + if (fuzzed_data_provider.ConsumeBool()) { + check_queue_control.Add(checks_2); + } + if (fuzzed_data_provider.ConsumeBool()) { + (void)check_queue_control.Wait(); + } +} diff --git a/src/test/fuzz/cuckoocache.cpp b/src/test/fuzz/cuckoocache.cpp new file mode 100644 index 0000000000..f674efe1b1 --- /dev/null +++ b/src/test/fuzz/cuckoocache.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <cuckoocache.h> +#include <optional.h> +#include <script/sigcache.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> + +#include <cstdint> +#include <string> +#include <vector> + +namespace { +FuzzedDataProvider* fuzzed_data_provider_ptr = nullptr; + +struct RandomHasher { + template <uint8_t> + uint32_t operator()(const bool& /* unused */) const + { + assert(fuzzed_data_provider_ptr != nullptr); + return fuzzed_data_provider_ptr->ConsumeIntegral<uint32_t>(); + } +}; +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + fuzzed_data_provider_ptr = &fuzzed_data_provider; + CuckooCache::cache<bool, RandomHasher> cuckoo_cache{}; + if (fuzzed_data_provider.ConsumeBool()) { + const size_t megabytes = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 16); + cuckoo_cache.setup_bytes(megabytes << 20); + } else { + cuckoo_cache.setup(fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(0, 4096)); + } + while (fuzzed_data_provider.ConsumeBool()) { + if (fuzzed_data_provider.ConsumeBool()) { + cuckoo_cache.insert(fuzzed_data_provider.ConsumeBool()); + } else { + cuckoo_cache.contains(fuzzed_data_provider.ConsumeBool(), fuzzed_data_provider.ConsumeBool()); + } + } + fuzzed_data_provider_ptr = nullptr; +} diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index 1e0abe94f8..001758ffdb 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 964fc85302..54793c890f 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/eval_script.cpp b/src/test/fuzz/eval_script.cpp index 6a1b037630..c556599db3 100644 --- a/src/test/fuzz/eval_script.cpp +++ b/src/test/fuzz/eval_script.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <pubkey.h> #include <script/interpreter.h> -#include <test/fuzz/fuzz.h> #include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> #include <util/memory.h> #include <limits> diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp new file mode 100644 index 0000000000..090994263e --- /dev/null +++ b/src/test/fuzz/fees.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <amount.h> +#include <optional.h> +#include <policy/fees.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const CFeeRate minimal_incremental_fee{ConsumeMoney(fuzzed_data_provider)}; + FeeFilterRounder fee_filter_rounder{minimal_incremental_fee}; + while (fuzzed_data_provider.ConsumeBool()) { + const CAmount current_minimum_fee = ConsumeMoney(fuzzed_data_provider); + const CAmount rounded_fee = fee_filter_rounder.round(current_minimum_fee); + assert(MoneyRange(rounded_fee)); + } +} diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index a085e36911..6e2188fe86 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp new file mode 100644 index 0000000000..3e20416116 --- /dev/null +++ b/src/test/fuzz/golomb_rice.cpp @@ -0,0 +1,112 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <blockfilter.h> +#include <serialize.h> +#include <streams.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/util.h> +#include <util/bytevectorhash.h> +#include <util/golombrice.h> + +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <iosfwd> +#include <unordered_set> +#include <vector> + +namespace { +uint64_t MapIntoRange(const uint64_t x, const uint64_t n) +{ + const uint64_t x_hi = x >> 32; + const uint64_t x_lo = x & 0xFFFFFFFF; + const uint64_t n_hi = n >> 32; + const uint64_t n_lo = n & 0xFFFFFFFF; + const uint64_t ac = x_hi * n_hi; + const uint64_t ad = x_hi * n_lo; + const uint64_t bc = x_lo * n_hi; + const uint64_t bd = x_lo * n_lo; + const uint64_t mid34 = (bd >> 32) + (bc & 0xFFFFFFFF) + (ad & 0xFFFFFFFF); + const uint64_t upper64 = ac + (bc >> 32) + (ad >> 32) + (mid34 >> 32); + return upper64; +} + +uint64_t HashToRange(const std::vector<uint8_t>& element, const uint64_t f) +{ + const uint64_t hash = CSipHasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL) + .Write(element.data(), element.size()) + .Finalize(); + return MapIntoRange(hash, f); +} + +std::vector<uint64_t> BuildHashedSet(const std::unordered_set<std::vector<uint8_t>, ByteVectorHash>& elements, const uint64_t f) +{ + std::vector<uint64_t> hashed_elements; + hashed_elements.reserve(elements.size()); + for (const std::vector<uint8_t>& element : elements) { + hashed_elements.push_back(HashToRange(element, f)); + } + std::sort(hashed_elements.begin(), hashed_elements.end()); + return hashed_elements; +} +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + std::vector<uint8_t> golomb_rice_data; + std::vector<uint64_t> encoded_deltas; + { + std::unordered_set<std::vector<uint8_t>, ByteVectorHash> elements; + const int n = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 512); + for (int i = 0; i < n; ++i) { + elements.insert(ConsumeRandomLengthByteVector(fuzzed_data_provider, 16)); + } + CVectorWriter stream(SER_NETWORK, 0, golomb_rice_data, 0); + WriteCompactSize(stream, static_cast<uint32_t>(elements.size())); + BitStreamWriter<CVectorWriter> bitwriter(stream); + if (!elements.empty()) { + uint64_t last_value = 0; + for (const uint64_t value : BuildHashedSet(elements, static_cast<uint64_t>(elements.size()) * static_cast<uint64_t>(BASIC_FILTER_M))) { + const uint64_t delta = value - last_value; + encoded_deltas.push_back(delta); + GolombRiceEncode(bitwriter, BASIC_FILTER_P, delta); + last_value = value; + } + } + bitwriter.Flush(); + } + + std::vector<uint64_t> decoded_deltas; + { + VectorReader stream{SER_NETWORK, 0, golomb_rice_data, 0}; + BitStreamReader<VectorReader> bitreader(stream); + const uint32_t n = static_cast<uint32_t>(ReadCompactSize(stream)); + for (uint32_t i = 0; i < n; ++i) { + decoded_deltas.push_back(GolombRiceDecode(bitreader, BASIC_FILTER_P)); + } + } + + assert(encoded_deltas == decoded_deltas); + + { + const std::vector<uint8_t> random_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider, 1024); + VectorReader stream{SER_NETWORK, 0, random_bytes, 0}; + uint32_t n; + try { + n = static_cast<uint32_t>(ReadCompactSize(stream)); + } catch (const std::ios_base::failure&) { + return; + } + BitStreamReader<VectorReader> bitreader(stream); + for (uint32_t i = 0; i < std::min<uint32_t>(n, 1024); ++i) { + try { + (void)GolombRiceDecode(bitreader, BASIC_FILTER_P); + } catch (const std::ios_base::failure&) { + } + } + } +} diff --git a/src/test/fuzz/hex.cpp b/src/test/fuzz/hex.cpp index 3bbf0084c2..5fed17c17c 100644 --- a/src/test/fuzz/hex.cpp +++ b/src/test/fuzz/hex.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <core_io.h> -#include <pubkey.h> #include <primitives/block.h> +#include <pubkey.h> #include <rpc/util.h> #include <test/fuzz/fuzz.h> #include <uint256.h> diff --git a/src/test/fuzz/http_request.cpp b/src/test/fuzz/http_request.cpp new file mode 100644 index 0000000000..ebf89749e9 --- /dev/null +++ b/src/test/fuzz/http_request.cpp @@ -0,0 +1,73 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <httpserver.h> +#include <netaddress.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <event2/buffer.h> +#include <event2/event.h> +#include <event2/http.h> +#include <event2/http_struct.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +// workaround for libevent versions before 2.1.1, +// when internal functions didn't have underscores at the end +#if LIBEVENT_VERSION_NUMBER < 0x02010100 +extern "C" int evhttp_parse_firstline(struct evhttp_request*, struct evbuffer*); +extern "C" int evhttp_parse_headers(struct evhttp_request*, struct evbuffer*); +inline int evhttp_parse_firstline_(struct evhttp_request* r, struct evbuffer* b) +{ + return evhttp_parse_firstline(r, b); +} +inline int evhttp_parse_headers_(struct evhttp_request* r, struct evbuffer* b) +{ + return evhttp_parse_headers(r, b); +} +#else +extern "C" int evhttp_parse_firstline_(struct evhttp_request*, struct evbuffer*); +extern "C" int evhttp_parse_headers_(struct evhttp_request*, struct evbuffer*); +#endif + +std::string RequestMethodString(HTTPRequest::RequestMethod m); + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + evhttp_request* evreq = evhttp_request_new(nullptr, nullptr); + assert(evreq != nullptr); + evreq->kind = EVHTTP_REQUEST; + evbuffer* evbuf = evbuffer_new(); + assert(evbuf != nullptr); + const std::vector<uint8_t> http_buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, 4096); + evbuffer_add(evbuf, http_buffer.data(), http_buffer.size()); + if (evhttp_parse_firstline_(evreq, evbuf) != 1 || evhttp_parse_headers_(evreq, evbuf) != 1) { + evbuffer_free(evbuf); + evhttp_request_free(evreq); + return; + } + + HTTPRequest http_request{evreq, true}; + const HTTPRequest::RequestMethod request_method = http_request.GetRequestMethod(); + (void)RequestMethodString(request_method); + (void)http_request.GetURI(); + (void)http_request.GetHeader("Host"); + const std::string header = fuzzed_data_provider.ConsumeRandomLengthString(16); + (void)http_request.GetHeader(header); + (void)http_request.WriteHeader(header, fuzzed_data_provider.ConsumeRandomLengthString(16)); + (void)http_request.GetHeader(header); + const std::string body = http_request.ReadBody(); + assert(body.empty()); + const CService service = http_request.GetPeer(); + assert(service.ToString() == "[::]:0"); + + evbuffer_free(evbuf); + evhttp_request_free(evreq); +} diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index fff2fabd17..9dbf0fcc90 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,6 +23,7 @@ #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> #include <time.h> #include <uint256.h> #include <util/moneystr.h> @@ -35,6 +36,7 @@ #include <cassert> #include <chrono> #include <limits> +#include <set> #include <vector> void initialize() @@ -90,8 +92,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) } (void)GetSizeOfCompactSize(u64); (void)GetSpecialScriptSize(u32); - // (void)GetVirtualTransactionSize(i64, i64); // function defined only for a subset of int64_t inputs - // (void)GetVirtualTransactionSize(i64, i64, u32); // function defined only for a subset of int64_t/uint32_t inputs + if (!MultiplicationOverflow(i64, static_cast<int64_t>(::nBytesPerSigOp)) && !AdditionOverflow(i64 * ::nBytesPerSigOp, static_cast<int64_t>(4))) { + (void)GetVirtualTransactionSize(i64, i64); + } + if (!MultiplicationOverflow(i64, static_cast<int64_t>(u32)) && !AdditionOverflow(i64, static_cast<int64_t>(4)) && !AdditionOverflow(i64 * u32, static_cast<int64_t>(4))) { + (void)GetVirtualTransactionSize(i64, i64, u32); + } (void)HexDigit(ch); (void)MoneyRange(i64); (void)ToString(i64); @@ -109,6 +115,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)memusage::DynamicUsage(u8); const unsigned char uch = static_cast<unsigned char>(u8); (void)memusage::DynamicUsage(uch); + { + const std::set<int64_t> i64s{i64, static_cast<int64_t>(u64)}; + const size_t dynamic_usage = memusage::DynamicUsage(i64s); + const size_t incremental_dynamic_usage = memusage::IncrementalDynamicUsage(i64s); + assert(dynamic_usage == incremental_dynamic_usage * i64s.size()); + } (void)MillisToTimeval(i64); const double d = ser_uint64_to_double(u64); assert(ser_double_to_uint64(d) == u64); diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 09580c0c52..3597f51e51 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -17,8 +17,7 @@ namespace { const std::string locale_identifiers[] = { - "C", "C.UTF-8", "aa_DJ", "aa_DJ.ISO-8859-1", "aa_DJ.UTF-8", "aa_ER", "aa_ER.UTF-8", "aa_ET", "aa_ET.UTF-8", "af_ZA", "af_ZA.ISO-8859-1", "af_ZA.UTF-8", "agr_PE", "agr_PE.UTF-8", "ak_GH", "ak_GH.UTF-8", "am_ET", "am_ET.UTF-8", "an_ES", "an_ES.ISO-8859-15", "an_ES.UTF-8", "anp_IN", "anp_IN.UTF-8", "ar_AE", "ar_AE.ISO-8859-6", "ar_AE.UTF-8", "ar_BH", "ar_BH.ISO-8859-6", "ar_BH.UTF-8", "ar_DZ", "ar_DZ.ISO-8859-6", "ar_DZ.UTF-8", "ar_EG", "ar_EG.ISO-8859-6", "ar_EG.UTF-8", "ar_IN", "ar_IN.UTF-8", "ar_IQ", "ar_IQ.ISO-8859-6", "ar_IQ.UTF-8", "ar_JO", "ar_JO.ISO-8859-6", "ar_JO.UTF-8", "ar_KW", "ar_KW.ISO-8859-6", "ar_KW.UTF-8", "ar_LB", "ar_LB.ISO-8859-6", "ar_LB.UTF-8", "ar_LY", "ar_LY.ISO-8859-6", "ar_LY.UTF-8", "ar_MA", "ar_MA.ISO-8859-6", "ar_MA.UTF-8", "ar_OM", "ar_OM.ISO-8859-6", "ar_OM.UTF-8", "ar_QA", "ar_QA.ISO-8859-6", "ar_QA.UTF-8", "ar_SA", "ar_SA.ISO-8859-6", "ar_SA.UTF-8", "ar_SD", "ar_SD.ISO-8859-6", "ar_SD.UTF-8", "ar_SS", "ar_SS.UTF-8", "ar_SY", "ar_SY.ISO-8859-6", "ar_SY.UTF-8", "ar_TN", "ar_TN.ISO-8859-6", "ar_TN.UTF-8", "ar_YE", "ar_YE.ISO-8859-6", "ar_YE.UTF-8", "as_IN", "as_IN.UTF-8", "ast_ES", "ast_ES.ISO-8859-15", "ast_ES.UTF-8", "ayc_PE", "ayc_PE.UTF-8", "az_AZ", "az_AZ.UTF-8", "az_IR", "az_IR.UTF-8", "be_BY", "be_BY.CP1251", "be_BY.UTF-8", "bem_ZM", "bem_ZM.UTF-8", "ber_DZ", "ber_DZ.UTF-8", "ber_MA", "ber_MA.UTF-8", "bg_BG", "bg_BG.CP1251", "bg_BG.UTF-8", "bho_IN", "bho_IN.UTF-8", "bho_NP", "bho_NP.UTF-8", "bi_VU", "bi_VU.UTF-8", "bn_BD", "bn_BD.UTF-8", "bn_IN", "bn_IN.UTF-8", "bo_CN", "bo_CN.UTF-8", "bo_IN", "bo_IN.UTF-8", "br_FR", "br_FR.ISO-8859-1", "br_FR.UTF-8", "brx_IN", "brx_IN.UTF-8", "bs_BA", "bs_BA.ISO-8859-2", "bs_BA.UTF-8", "byn_ER", "byn_ER.UTF-8", "ca_AD", "ca_AD.ISO-8859-15", "ca_AD.UTF-8", "ca_ES", "ca_ES.ISO-8859-1", "ca_ES.UTF-8", "ca_FR", "ca_FR.ISO-8859-15", "ca_FR.UTF-8", "ca_IT", "ca_IT.ISO-8859-15", "ca_IT.UTF-8", "ce_RU", "ce_RU.UTF-8", "chr_US", "chr_US.UTF-8", "ckb_IQ", "ckb_IQ.UTF-8", "cmn_TW", "cmn_TW.UTF-8", "crh_UA", "crh_UA.UTF-8", "csb_PL", "csb_PL.UTF-8", "cs_CZ", "cs_CZ.ISO-8859-2", "cs_CZ.UTF-8", "cv_RU", "cv_RU.UTF-8", "cy_GB", "cy_GB.ISO-8859-14", "cy_GB.UTF-8", "da_DK", "da_DK.ISO-8859-1", "da_DK.UTF-8", "de_AT", "de_AT.ISO-8859-1", "de_AT.UTF-8", "de_BE", "de_BE.ISO-8859-1", "de_BE.UTF-8", "de_CH", "de_CH.ISO-8859-1", "de_CH.UTF-8", "de_DE", "de_DE.ISO-8859-1", "de_DE.UTF-8", "de_IT", "de_IT.ISO-8859-1", "de_IT.UTF-8", "de_LU", "de_LU.ISO-8859-1", "de_LU.UTF-8", "doi_IN", "doi_IN.UTF-8", "dv_MV", "dv_MV.UTF-8", "dz_BT", "dz_BT.UTF-8", "el_CY", "el_CY.ISO-8859-7", "el_CY.UTF-8", "el_GR", "el_GR.ISO-8859-7", "el_GR.UTF-8", "en_AG", "en_AG.UTF-8", "en_AU", "en_AU.ISO-8859-1", "en_AU.UTF-8", "en_BW", "en_BW.ISO-8859-1", "en_BW.UTF-8", "en_CA", "en_CA.ISO-8859-1", "en_CA.UTF-8", "en_DK", "en_DK.ISO-8859-1", "en_DK.ISO-8859-15", "en_DK.UTF-8", "en_GB", "en_GB.ISO-8859-1", "en_GB.ISO-8859-15", "en_GB.UTF-8", "en_HK", "en_HK.ISO-8859-1", "en_HK.UTF-8", "en_IE", "en_IE.ISO-8859-1", "en_IE.UTF-8", "en_IL", "en_IL.UTF-8", "en_IN", "en_IN.UTF-8", "en_NG", "en_NG.UTF-8", "en_NZ", "en_NZ.ISO-8859-1", "en_NZ.UTF-8", "en_PH", "en_PH.ISO-8859-1", "en_PH.UTF-8", "en_SG", "en_SG.ISO-8859-1", "en_SG.UTF-8", "en_US", "en_US.ISO-8859-1", "en_US.ISO-8859-15", "en_US.UTF-8", "en_ZA", "en_ZA.ISO-8859-1", "en_ZA.UTF-8", "en_ZM", "en_ZM.UTF-8", "en_ZW", "en_ZW.ISO-8859-1", "en_ZW.UTF-8", "es_AR", "es_AR.ISO-8859-1", "es_AR.UTF-8", "es_BO", "es_BO.ISO-8859-1", "es_BO.UTF-8", "es_CL", "es_CL.ISO-8859-1", "es_CL.UTF-8", "es_CO", "es_CO.ISO-8859-1", "es_CO.UTF-8", "es_CR", "es_CR.ISO-8859-1", "es_CR.UTF-8", "es_CU", "es_CU.UTF-8", "es_DO", "es_DO.ISO-8859-1", "es_DO.UTF-8", "es_EC", "es_EC.ISO-8859-1", "es_EC.UTF-8", "es_ES", "es_ES.ISO-8859-1", "es_ES.UTF-8", "es_GT", "es_GT.ISO-8859-1", "es_GT.UTF-8", "es_HN", "es_HN.ISO-8859-1", "es_HN.UTF-8", "es_MX", "es_MX.ISO-8859-1", "es_MX.UTF-8", "es_NI", "es_NI.ISO-8859-1", "es_NI.UTF-8", "es_PA", "es_PA.ISO-8859-1", "es_PA.UTF-8", "es_PE", "es_PE.ISO-8859-1", "es_PE.UTF-8", "es_PR", "es_PR.ISO-8859-1", "es_PR.UTF-8", "es_PY", "es_PY.ISO-8859-1", "es_PY.UTF-8", "es_SV", "es_SV.ISO-8859-1", "es_SV.UTF-8", "es_US", "es_US.ISO-8859-1", "es_US.UTF-8", "es_UY", "es_UY.ISO-8859-1", "es_UY.UTF-8", "es_VE", "es_VE.ISO-8859-1", "es_VE.UTF-8", "et_EE", "et_EE.ISO-8859-1", "et_EE.ISO-8859-15", "et_EE.UTF-8", "eu_ES", "eu_ES.ISO-8859-1", "eu_ES.UTF-8", "eu_FR", "eu_FR.ISO-8859-1", "eu_FR.UTF-8", "fa_IR", "fa_IR.UTF-8", "ff_SN", "ff_SN.UTF-8", "fi_FI", "fi_FI.ISO-8859-1", "fi_FI.UTF-8", "fil_PH", "fil_PH.UTF-8", "fo_FO", "fo_FO.ISO-8859-1", "fo_FO.UTF-8", "fr_BE", "fr_BE.ISO-8859-1", "fr_BE.UTF-8", "fr_CA", "fr_CA.ISO-8859-1", "fr_CA.UTF-8", "fr_CH", "fr_CH.ISO-8859-1", "fr_CH.UTF-8", "fr_FR", "fr_FR.ISO-8859-1", "fr_FR.UTF-8", "fr_LU", "fr_LU.ISO-8859-1", "fr_LU.UTF-8", "fur_IT", "fur_IT.UTF-8", "fy_DE", "fy_DE.UTF-8", "fy_NL", "fy_NL.UTF-8", "ga_IE", "ga_IE.ISO-8859-1", "ga_IE.UTF-8", "gd_GB", "gd_GB.ISO-8859-15", "gd_GB.UTF-8", "gez_ER", "gez_ER.UTF-8", "gez_ET", "gez_ET.UTF-8", "gl_ES", "gl_ES.ISO-8859-1", "gl_ES.UTF-8", "gu_IN", "gu_IN.UTF-8", "gv_GB", "gv_GB.ISO-8859-1", "gv_GB.UTF-8", "hak_TW", "hak_TW.UTF-8", "ha_NG", "ha_NG.UTF-8", "he_IL", "he_IL.ISO-8859-8", "he_IL.UTF-8", "hif_FJ", "hif_FJ.UTF-8", "hi_IN", "hi_IN.UTF-8", "hne_IN", "hne_IN.UTF-8", "hr_HR", "hr_HR.ISO-8859-2", "hr_HR.UTF-8", "hsb_DE", "hsb_DE.ISO-8859-2", "hsb_DE.UTF-8", "ht_HT", "ht_HT.UTF-8", "hu_HU", "hu_HU.ISO-8859-2", "hu_HU.UTF-8", "hy_AM", "hy_AM.ARMSCII-8", "hy_AM.UTF-8", "ia_FR", "ia_FR.UTF-8", "id_ID", "id_ID.ISO-8859-1", "id_ID.UTF-8", "ig_NG", "ig_NG.UTF-8", "ik_CA", "ik_CA.UTF-8", "is_IS", "is_IS.ISO-8859-1", "is_IS.UTF-8", "it_CH", "it_CH.ISO-8859-1", "it_CH.UTF-8", "it_IT", "it_IT.ISO-8859-1", "it_IT.UTF-8", "iu_CA", "iu_CA.UTF-8", "kab_DZ", "kab_DZ.UTF-8", "ka_GE", "ka_GE.GEORGIAN-PS", "ka_GE.UTF-8", "kk_KZ", "kk_KZ.PT154", "kk_KZ.RK1048", "kk_KZ.UTF-8", "kl_GL", "kl_GL.ISO-8859-1", "kl_GL.UTF-8", "km_KH", "km_KH.UTF-8", "kn_IN", "kn_IN.UTF-8", "kok_IN", "kok_IN.UTF-8", "ks_IN", "ks_IN.UTF-8", "ku_TR", "ku_TR.ISO-8859-9", "ku_TR.UTF-8", "kw_GB", "kw_GB.ISO-8859-1", "kw_GB.UTF-8", "ky_KG", "ky_KG.UTF-8", "lb_LU", "lb_LU.UTF-8", "lg_UG", "lg_UG.ISO-8859-10", "lg_UG.UTF-8", "li_BE", "li_BE.UTF-8", "lij_IT", "lij_IT.UTF-8", "li_NL", "li_NL.UTF-8", "ln_CD", "ln_CD.UTF-8", "lo_LA", "lo_LA.UTF-8", "lt_LT", "lt_LT.ISO-8859-13", "lt_LT.UTF-8", "lv_LV", "lv_LV.ISO-8859-13", "lv_LV.UTF-8", "lzh_TW", "lzh_TW.UTF-8", "mag_IN", "mag_IN.UTF-8", "mai_IN", "mai_IN.UTF-8", "mai_NP", "mai_NP.UTF-8", "mfe_MU", "mfe_MU.UTF-8", "mg_MG", "mg_MG.ISO-8859-15", "mg_MG.UTF-8", "mhr_RU", "mhr_RU.UTF-8", "mi_NZ", "mi_NZ.ISO-8859-13", "mi_NZ.UTF-8", "miq_NI", "miq_NI.UTF-8", "mjw_IN", "mjw_IN.UTF-8", "mk_MK", "mk_MK.ISO-8859-5", "mk_MK.UTF-8", "ml_IN", "ml_IN.UTF-8", "mni_IN", "mni_IN.UTF-8", "mn_MN", "mn_MN.UTF-8", "mr_IN", "mr_IN.UTF-8", "ms_MY", "ms_MY.ISO-8859-1", "ms_MY.UTF-8", "mt_MT", "mt_MT.ISO-8859-3", "mt_MT.UTF-8", "my_MM", "my_MM.UTF-8", "nan_TW", "nan_TW.UTF-8", "nb_NO", "nb_NO.ISO-8859-1", "nb_NO.UTF-8", "nds_DE", "nds_DE.UTF-8", "nds_NL", "nds_NL.UTF-8", "ne_NP", "ne_NP.UTF-8", "nhn_MX", "nhn_MX.UTF-8", "niu_NU", "niu_NU.UTF-8", "niu_NZ", "niu_NZ.UTF-8", "nl_AW", "nl_AW.UTF-8", "nl_BE", "nl_BE.ISO-8859-1", "nl_BE.UTF-8", "nl_NL", "nl_NL.ISO-8859-1", "nl_NL.UTF-8", "nn_NO", "nn_NO.ISO-8859-1", "nn_NO.UTF-8", "nr_ZA", "nr_ZA.UTF-8", "nso_ZA", "nso_ZA.UTF-8", "oc_FR", "oc_FR.ISO-8859-1", "oc_FR.UTF-8", "om_ET", "om_ET.UTF-8", "om_KE", "om_KE.ISO-8859-1", "om_KE.UTF-8", "or_IN", "or_IN.UTF-8", "os_RU", "os_RU.UTF-8", "pa_IN", "pa_IN.UTF-8", "pap_AW", "pap_AW.UTF-8", "pap_CW", "pap_CW.UTF-8", "pa_PK", "pa_PK.UTF-8", "pl_PL", "pl_PL.ISO-8859-2", "pl_PL.UTF-8", "ps_AF", "ps_AF.UTF-8", "pt_BR", "pt_BR.ISO-8859-1", "pt_BR.UTF-8", "pt_PT", "pt_PT.ISO-8859-1", "pt_PT.UTF-8", "quz_PE", "quz_PE.UTF-8", "raj_IN", "raj_IN.UTF-8", "ro_RO", "ro_RO.ISO-8859-2", "ro_RO.UTF-8", "ru_RU", "ru_RU.CP1251", "ru_RU.ISO-8859-5", "ru_RU.KOI8-R", "ru_RU.UTF-8", "ru_UA", "ru_UA.KOI8-U", "ru_UA.UTF-8", "rw_RW", "rw_RW.UTF-8", "sa_IN", "sa_IN.UTF-8", "sat_IN", "sat_IN.UTF-8", "sc_IT", "sc_IT.UTF-8", "sd_IN", "sd_IN.UTF-8", "sd_PK", "sd_PK.UTF-8", "se_NO", "se_NO.UTF-8", "sgs_LT", "sgs_LT.UTF-8", "shn_MM", "shn_MM.UTF-8", "shs_CA", "shs_CA.UTF-8", "sid_ET", "sid_ET.UTF-8", "si_LK", "si_LK.UTF-8", "sk_SK", "sk_SK.ISO-8859-2", "sk_SK.UTF-8", "sl_SI", "sl_SI.ISO-8859-2", "sl_SI.UTF-8", "sm_WS", "sm_WS.UTF-8", "so_DJ", "so_DJ.ISO-8859-1", "so_DJ.UTF-8", "so_ET", "so_ET.UTF-8", "so_KE", "so_KE.ISO-8859-1", "so_KE.UTF-8", "so_SO", "so_SO.ISO-8859-1", "so_SO.UTF-8", "sq_AL", "sq_AL.ISO-8859-1", "sq_AL.UTF-8", "sq_MK", "sq_MK.UTF-8", "sr_ME", "sr_ME.UTF-8", "sr_RS", "sr_RS.UTF-8", "ss_ZA", "ss_ZA.UTF-8", "st_ZA", "st_ZA.ISO-8859-1", "st_ZA.UTF-8", "sv_FI", "sv_FI.ISO-8859-1", "sv_FI.UTF-8", "sv_SE", "sv_SE.ISO-8859-1", "sv_SE.ISO-8859-15", "sv_SE.UTF-8", "sw_KE", "sw_KE.UTF-8", "sw_TZ", "sw_TZ.UTF-8", "szl_PL", "szl_PL.UTF-8", "ta_IN", "ta_IN.UTF-8", "ta_LK", "ta_LK.UTF-8", "te_IN", "te_IN.UTF-8", "tg_TJ", "tg_TJ.KOI8-T", "tg_TJ.UTF-8", "the_NP", "the_NP.UTF-8", "th_TH", "th_TH.TIS-620", "th_TH.UTF-8", "ti_ER", "ti_ER.UTF-8", "ti_ET", "ti_ET.UTF-8", "tig_ER", "tig_ER.UTF-8", "tk_TM", "tk_TM.UTF-8", "tl_PH", "tl_PH.ISO-8859-1", "tl_PH.UTF-8", "tn_ZA", "tn_ZA.UTF-8", "to_TO", "to_TO.UTF-8", "tpi_PG", "tpi_PG.UTF-8", "tr_CY", "tr_CY.ISO-8859-9", "tr_CY.UTF-8", "tr_TR", "tr_TR.ISO-8859-9", "tr_TR.UTF-8", "ts_ZA", "ts_ZA.UTF-8", "tt_RU", "tt_RU.UTF-8", "ug_CN", "ug_CN.UTF-8", "uk_UA", "uk_UA.KOI8-U", "uk_UA.UTF-8", "unm_US", "unm_US.UTF-8", "ur_IN", "ur_IN.UTF-8", "ur_PK", "ur_PK.UTF-8", "uz_UZ", "uz_UZ.ISO-8859-1", "uz_UZ.UTF-8", "ve_ZA", "ve_ZA.UTF-8", "vi_VN", "vi_VN.UTF-8", "wa_BE", "wa_BE.ISO-8859-1", "wa_BE.UTF-8", "wae_CH", "wae_CH.UTF-8", "wal_ET", "wal_ET.UTF-8", "wo_SN", "wo_SN.UTF-8", "xh_ZA", "xh_ZA.ISO-8859-1", "xh_ZA.UTF-8", "yi_US", "yi_US.CP1255", "yi_US.UTF-8", "yo_NG", "yo_NG.UTF-8", "yue_HK", "yue_HK.UTF-8", "yuw_PG", "yuw_PG.UTF-8", "zh_CN", "zh_CN.GB18030", "zh_CN.GB2312", "zh_CN.GBK", "zh_CN.UTF-8", "zh_HK", "zh_HK.BIG5-HKSCS", "zh_HK.UTF-8", "zh_SG", "zh_SG.GB2312", "zh_SG.GBK", "zh_SG.UTF-8", "zh_TW", "zh_TW.BIG5", "zh_TW.EUC-TW", "zh_TW.UTF-8", "zu_ZA", "zu_ZA.ISO-8859-1", "zu_ZA.UTF-8" -}; + "C", "C.UTF-8", "aa_DJ", "aa_DJ.ISO-8859-1", "aa_DJ.UTF-8", "aa_ER", "aa_ER.UTF-8", "aa_ET", "aa_ET.UTF-8", "af_ZA", "af_ZA.ISO-8859-1", "af_ZA.UTF-8", "agr_PE", "agr_PE.UTF-8", "ak_GH", "ak_GH.UTF-8", "am_ET", "am_ET.UTF-8", "an_ES", "an_ES.ISO-8859-15", "an_ES.UTF-8", "anp_IN", "anp_IN.UTF-8", "ar_AE", "ar_AE.ISO-8859-6", "ar_AE.UTF-8", "ar_BH", "ar_BH.ISO-8859-6", "ar_BH.UTF-8", "ar_DZ", "ar_DZ.ISO-8859-6", "ar_DZ.UTF-8", "ar_EG", "ar_EG.ISO-8859-6", "ar_EG.UTF-8", "ar_IN", "ar_IN.UTF-8", "ar_IQ", "ar_IQ.ISO-8859-6", "ar_IQ.UTF-8", "ar_JO", "ar_JO.ISO-8859-6", "ar_JO.UTF-8", "ar_KW", "ar_KW.ISO-8859-6", "ar_KW.UTF-8", "ar_LB", "ar_LB.ISO-8859-6", "ar_LB.UTF-8", "ar_LY", "ar_LY.ISO-8859-6", "ar_LY.UTF-8", "ar_MA", "ar_MA.ISO-8859-6", "ar_MA.UTF-8", "ar_OM", "ar_OM.ISO-8859-6", "ar_OM.UTF-8", "ar_QA", "ar_QA.ISO-8859-6", "ar_QA.UTF-8", "ar_SA", "ar_SA.ISO-8859-6", "ar_SA.UTF-8", "ar_SD", "ar_SD.ISO-8859-6", "ar_SD.UTF-8", "ar_SS", "ar_SS.UTF-8", "ar_SY", "ar_SY.ISO-8859-6", "ar_SY.UTF-8", "ar_TN", "ar_TN.ISO-8859-6", "ar_TN.UTF-8", "ar_YE", "ar_YE.ISO-8859-6", "ar_YE.UTF-8", "as_IN", "as_IN.UTF-8", "ast_ES", "ast_ES.ISO-8859-15", "ast_ES.UTF-8", "ayc_PE", "ayc_PE.UTF-8", "az_AZ", "az_AZ.UTF-8", "az_IR", "az_IR.UTF-8", "be_BY", "be_BY.CP1251", "be_BY.UTF-8", "bem_ZM", "bem_ZM.UTF-8", "ber_DZ", "ber_DZ.UTF-8", "ber_MA", "ber_MA.UTF-8", "bg_BG", "bg_BG.CP1251", "bg_BG.UTF-8", "bho_IN", "bho_IN.UTF-8", "bho_NP", "bho_NP.UTF-8", "bi_VU", "bi_VU.UTF-8", "bn_BD", "bn_BD.UTF-8", "bn_IN", "bn_IN.UTF-8", "bo_CN", "bo_CN.UTF-8", "bo_IN", "bo_IN.UTF-8", "br_FR", "br_FR.ISO-8859-1", "br_FR.UTF-8", "brx_IN", "brx_IN.UTF-8", "bs_BA", "bs_BA.ISO-8859-2", "bs_BA.UTF-8", "byn_ER", "byn_ER.UTF-8", "ca_AD", "ca_AD.ISO-8859-15", "ca_AD.UTF-8", "ca_ES", "ca_ES.ISO-8859-1", "ca_ES.UTF-8", "ca_FR", "ca_FR.ISO-8859-15", "ca_FR.UTF-8", "ca_IT", "ca_IT.ISO-8859-15", "ca_IT.UTF-8", "ce_RU", "ce_RU.UTF-8", "chr_US", "chr_US.UTF-8", "ckb_IQ", "ckb_IQ.UTF-8", "cmn_TW", "cmn_TW.UTF-8", "crh_UA", "crh_UA.UTF-8", "csb_PL", "csb_PL.UTF-8", "cs_CZ", "cs_CZ.ISO-8859-2", "cs_CZ.UTF-8", "cv_RU", "cv_RU.UTF-8", "cy_GB", "cy_GB.ISO-8859-14", "cy_GB.UTF-8", "da_DK", "da_DK.ISO-8859-1", "da_DK.UTF-8", "de_AT", "de_AT.ISO-8859-1", "de_AT.UTF-8", "de_BE", "de_BE.ISO-8859-1", "de_BE.UTF-8", "de_CH", "de_CH.ISO-8859-1", "de_CH.UTF-8", "de_DE", "de_DE.ISO-8859-1", "de_DE.UTF-8", "de_IT", "de_IT.ISO-8859-1", "de_IT.UTF-8", "de_LU", "de_LU.ISO-8859-1", "de_LU.UTF-8", "doi_IN", "doi_IN.UTF-8", "dv_MV", "dv_MV.UTF-8", "dz_BT", "dz_BT.UTF-8", "el_CY", "el_CY.ISO-8859-7", "el_CY.UTF-8", "el_GR", "el_GR.ISO-8859-7", "el_GR.UTF-8", "en_AG", "en_AG.UTF-8", "en_AU", "en_AU.ISO-8859-1", "en_AU.UTF-8", "en_BW", "en_BW.ISO-8859-1", "en_BW.UTF-8", "en_CA", "en_CA.ISO-8859-1", "en_CA.UTF-8", "en_DK", "en_DK.ISO-8859-1", "en_DK.ISO-8859-15", "en_DK.UTF-8", "en_GB", "en_GB.ISO-8859-1", "en_GB.ISO-8859-15", "en_GB.UTF-8", "en_HK", "en_HK.ISO-8859-1", "en_HK.UTF-8", "en_IE", "en_IE.ISO-8859-1", "en_IE.UTF-8", "en_IL", "en_IL.UTF-8", "en_IN", "en_IN.UTF-8", "en_NG", "en_NG.UTF-8", "en_NZ", "en_NZ.ISO-8859-1", "en_NZ.UTF-8", "en_PH", "en_PH.ISO-8859-1", "en_PH.UTF-8", "en_SG", "en_SG.ISO-8859-1", "en_SG.UTF-8", "en_US", "en_US.ISO-8859-1", "en_US.ISO-8859-15", "en_US.UTF-8", "en_ZA", "en_ZA.ISO-8859-1", "en_ZA.UTF-8", "en_ZM", "en_ZM.UTF-8", "en_ZW", "en_ZW.ISO-8859-1", "en_ZW.UTF-8", "es_AR", "es_AR.ISO-8859-1", "es_AR.UTF-8", "es_BO", "es_BO.ISO-8859-1", "es_BO.UTF-8", "es_CL", "es_CL.ISO-8859-1", "es_CL.UTF-8", "es_CO", "es_CO.ISO-8859-1", "es_CO.UTF-8", "es_CR", "es_CR.ISO-8859-1", "es_CR.UTF-8", "es_CU", "es_CU.UTF-8", "es_DO", "es_DO.ISO-8859-1", "es_DO.UTF-8", "es_EC", "es_EC.ISO-8859-1", "es_EC.UTF-8", "es_ES", "es_ES.ISO-8859-1", "es_ES.UTF-8", "es_GT", "es_GT.ISO-8859-1", "es_GT.UTF-8", "es_HN", "es_HN.ISO-8859-1", "es_HN.UTF-8", "es_MX", "es_MX.ISO-8859-1", "es_MX.UTF-8", "es_NI", "es_NI.ISO-8859-1", "es_NI.UTF-8", "es_PA", "es_PA.ISO-8859-1", "es_PA.UTF-8", "es_PE", "es_PE.ISO-8859-1", "es_PE.UTF-8", "es_PR", "es_PR.ISO-8859-1", "es_PR.UTF-8", "es_PY", "es_PY.ISO-8859-1", "es_PY.UTF-8", "es_SV", "es_SV.ISO-8859-1", "es_SV.UTF-8", "es_US", "es_US.ISO-8859-1", "es_US.UTF-8", "es_UY", "es_UY.ISO-8859-1", "es_UY.UTF-8", "es_VE", "es_VE.ISO-8859-1", "es_VE.UTF-8", "et_EE", "et_EE.ISO-8859-1", "et_EE.ISO-8859-15", "et_EE.UTF-8", "eu_ES", "eu_ES.ISO-8859-1", "eu_ES.UTF-8", "eu_FR", "eu_FR.ISO-8859-1", "eu_FR.UTF-8", "fa_IR", "fa_IR.UTF-8", "ff_SN", "ff_SN.UTF-8", "fi_FI", "fi_FI.ISO-8859-1", "fi_FI.UTF-8", "fil_PH", "fil_PH.UTF-8", "fo_FO", "fo_FO.ISO-8859-1", "fo_FO.UTF-8", "fr_BE", "fr_BE.ISO-8859-1", "fr_BE.UTF-8", "fr_CA", "fr_CA.ISO-8859-1", "fr_CA.UTF-8", "fr_CH", "fr_CH.ISO-8859-1", "fr_CH.UTF-8", "fr_FR", "fr_FR.ISO-8859-1", "fr_FR.UTF-8", "fr_LU", "fr_LU.ISO-8859-1", "fr_LU.UTF-8", "fur_IT", "fur_IT.UTF-8", "fy_DE", "fy_DE.UTF-8", "fy_NL", "fy_NL.UTF-8", "ga_IE", "ga_IE.ISO-8859-1", "ga_IE.UTF-8", "gd_GB", "gd_GB.ISO-8859-15", "gd_GB.UTF-8", "gez_ER", "gez_ER.UTF-8", "gez_ET", "gez_ET.UTF-8", "gl_ES", "gl_ES.ISO-8859-1", "gl_ES.UTF-8", "gu_IN", "gu_IN.UTF-8", "gv_GB", "gv_GB.ISO-8859-1", "gv_GB.UTF-8", "hak_TW", "hak_TW.UTF-8", "ha_NG", "ha_NG.UTF-8", "he_IL", "he_IL.ISO-8859-8", "he_IL.UTF-8", "hif_FJ", "hif_FJ.UTF-8", "hi_IN", "hi_IN.UTF-8", "hne_IN", "hne_IN.UTF-8", "hr_HR", "hr_HR.ISO-8859-2", "hr_HR.UTF-8", "hsb_DE", "hsb_DE.ISO-8859-2", "hsb_DE.UTF-8", "ht_HT", "ht_HT.UTF-8", "hu_HU", "hu_HU.ISO-8859-2", "hu_HU.UTF-8", "hy_AM", "hy_AM.ARMSCII-8", "hy_AM.UTF-8", "ia_FR", "ia_FR.UTF-8", "id_ID", "id_ID.ISO-8859-1", "id_ID.UTF-8", "ig_NG", "ig_NG.UTF-8", "ik_CA", "ik_CA.UTF-8", "is_IS", "is_IS.ISO-8859-1", "is_IS.UTF-8", "it_CH", "it_CH.ISO-8859-1", "it_CH.UTF-8", "it_IT", "it_IT.ISO-8859-1", "it_IT.UTF-8", "iu_CA", "iu_CA.UTF-8", "kab_DZ", "kab_DZ.UTF-8", "ka_GE", "ka_GE.GEORGIAN-PS", "ka_GE.UTF-8", "kk_KZ", "kk_KZ.PT154", "kk_KZ.RK1048", "kk_KZ.UTF-8", "kl_GL", "kl_GL.ISO-8859-1", "kl_GL.UTF-8", "km_KH", "km_KH.UTF-8", "kn_IN", "kn_IN.UTF-8", "kok_IN", "kok_IN.UTF-8", "ks_IN", "ks_IN.UTF-8", "ku_TR", "ku_TR.ISO-8859-9", "ku_TR.UTF-8", "kw_GB", "kw_GB.ISO-8859-1", "kw_GB.UTF-8", "ky_KG", "ky_KG.UTF-8", "lb_LU", "lb_LU.UTF-8", "lg_UG", "lg_UG.ISO-8859-10", "lg_UG.UTF-8", "li_BE", "li_BE.UTF-8", "lij_IT", "lij_IT.UTF-8", "li_NL", "li_NL.UTF-8", "ln_CD", "ln_CD.UTF-8", "lo_LA", "lo_LA.UTF-8", "lt_LT", "lt_LT.ISO-8859-13", "lt_LT.UTF-8", "lv_LV", "lv_LV.ISO-8859-13", "lv_LV.UTF-8", "lzh_TW", "lzh_TW.UTF-8", "mag_IN", "mag_IN.UTF-8", "mai_IN", "mai_IN.UTF-8", "mai_NP", "mai_NP.UTF-8", "mfe_MU", "mfe_MU.UTF-8", "mg_MG", "mg_MG.ISO-8859-15", "mg_MG.UTF-8", "mhr_RU", "mhr_RU.UTF-8", "mi_NZ", "mi_NZ.ISO-8859-13", "mi_NZ.UTF-8", "miq_NI", "miq_NI.UTF-8", "mjw_IN", "mjw_IN.UTF-8", "mk_MK", "mk_MK.ISO-8859-5", "mk_MK.UTF-8", "ml_IN", "ml_IN.UTF-8", "mni_IN", "mni_IN.UTF-8", "mn_MN", "mn_MN.UTF-8", "mr_IN", "mr_IN.UTF-8", "ms_MY", "ms_MY.ISO-8859-1", "ms_MY.UTF-8", "mt_MT", "mt_MT.ISO-8859-3", "mt_MT.UTF-8", "my_MM", "my_MM.UTF-8", "nan_TW", "nan_TW.UTF-8", "nb_NO", "nb_NO.ISO-8859-1", "nb_NO.UTF-8", "nds_DE", "nds_DE.UTF-8", "nds_NL", "nds_NL.UTF-8", "ne_NP", "ne_NP.UTF-8", "nhn_MX", "nhn_MX.UTF-8", "niu_NU", "niu_NU.UTF-8", "niu_NZ", "niu_NZ.UTF-8", "nl_AW", "nl_AW.UTF-8", "nl_BE", "nl_BE.ISO-8859-1", "nl_BE.UTF-8", "nl_NL", "nl_NL.ISO-8859-1", "nl_NL.UTF-8", "nn_NO", "nn_NO.ISO-8859-1", "nn_NO.UTF-8", "nr_ZA", "nr_ZA.UTF-8", "nso_ZA", "nso_ZA.UTF-8", "oc_FR", "oc_FR.ISO-8859-1", "oc_FR.UTF-8", "om_ET", "om_ET.UTF-8", "om_KE", "om_KE.ISO-8859-1", "om_KE.UTF-8", "or_IN", "or_IN.UTF-8", "os_RU", "os_RU.UTF-8", "pa_IN", "pa_IN.UTF-8", "pap_AW", "pap_AW.UTF-8", "pap_CW", "pap_CW.UTF-8", "pa_PK", "pa_PK.UTF-8", "pl_PL", "pl_PL.ISO-8859-2", "pl_PL.UTF-8", "ps_AF", "ps_AF.UTF-8", "pt_BR", "pt_BR.ISO-8859-1", "pt_BR.UTF-8", "pt_PT", "pt_PT.ISO-8859-1", "pt_PT.UTF-8", "quz_PE", "quz_PE.UTF-8", "raj_IN", "raj_IN.UTF-8", "ro_RO", "ro_RO.ISO-8859-2", "ro_RO.UTF-8", "ru_RU", "ru_RU.CP1251", "ru_RU.ISO-8859-5", "ru_RU.KOI8-R", "ru_RU.UTF-8", "ru_UA", "ru_UA.KOI8-U", "ru_UA.UTF-8", "rw_RW", "rw_RW.UTF-8", "sa_IN", "sa_IN.UTF-8", "sat_IN", "sat_IN.UTF-8", "sc_IT", "sc_IT.UTF-8", "sd_IN", "sd_IN.UTF-8", "sd_PK", "sd_PK.UTF-8", "se_NO", "se_NO.UTF-8", "sgs_LT", "sgs_LT.UTF-8", "shn_MM", "shn_MM.UTF-8", "shs_CA", "shs_CA.UTF-8", "sid_ET", "sid_ET.UTF-8", "si_LK", "si_LK.UTF-8", "sk_SK", "sk_SK.ISO-8859-2", "sk_SK.UTF-8", "sl_SI", "sl_SI.ISO-8859-2", "sl_SI.UTF-8", "sm_WS", "sm_WS.UTF-8", "so_DJ", "so_DJ.ISO-8859-1", "so_DJ.UTF-8", "so_ET", "so_ET.UTF-8", "so_KE", "so_KE.ISO-8859-1", "so_KE.UTF-8", "so_SO", "so_SO.ISO-8859-1", "so_SO.UTF-8", "sq_AL", "sq_AL.ISO-8859-1", "sq_AL.UTF-8", "sq_MK", "sq_MK.UTF-8", "sr_ME", "sr_ME.UTF-8", "sr_RS", "sr_RS.UTF-8", "ss_ZA", "ss_ZA.UTF-8", "st_ZA", "st_ZA.ISO-8859-1", "st_ZA.UTF-8", "sv_FI", "sv_FI.ISO-8859-1", "sv_FI.UTF-8", "sv_SE", "sv_SE.ISO-8859-1", "sv_SE.ISO-8859-15", "sv_SE.UTF-8", "sw_KE", "sw_KE.UTF-8", "sw_TZ", "sw_TZ.UTF-8", "szl_PL", "szl_PL.UTF-8", "ta_IN", "ta_IN.UTF-8", "ta_LK", "ta_LK.UTF-8", "te_IN", "te_IN.UTF-8", "tg_TJ", "tg_TJ.KOI8-T", "tg_TJ.UTF-8", "the_NP", "the_NP.UTF-8", "th_TH", "th_TH.TIS-620", "th_TH.UTF-8", "ti_ER", "ti_ER.UTF-8", "ti_ET", "ti_ET.UTF-8", "tig_ER", "tig_ER.UTF-8", "tk_TM", "tk_TM.UTF-8", "tl_PH", "tl_PH.ISO-8859-1", "tl_PH.UTF-8", "tn_ZA", "tn_ZA.UTF-8", "to_TO", "to_TO.UTF-8", "tpi_PG", "tpi_PG.UTF-8", "tr_CY", "tr_CY.ISO-8859-9", "tr_CY.UTF-8", "tr_TR", "tr_TR.ISO-8859-9", "tr_TR.UTF-8", "ts_ZA", "ts_ZA.UTF-8", "tt_RU", "tt_RU.UTF-8", "ug_CN", "ug_CN.UTF-8", "uk_UA", "uk_UA.KOI8-U", "uk_UA.UTF-8", "unm_US", "unm_US.UTF-8", "ur_IN", "ur_IN.UTF-8", "ur_PK", "ur_PK.UTF-8", "uz_UZ", "uz_UZ.ISO-8859-1", "uz_UZ.UTF-8", "ve_ZA", "ve_ZA.UTF-8", "vi_VN", "vi_VN.UTF-8", "wa_BE", "wa_BE.ISO-8859-1", "wa_BE.UTF-8", "wae_CH", "wae_CH.UTF-8", "wal_ET", "wal_ET.UTF-8", "wo_SN", "wo_SN.UTF-8", "xh_ZA", "xh_ZA.ISO-8859-1", "xh_ZA.UTF-8", "yi_US", "yi_US.CP1255", "yi_US.UTF-8", "yo_NG", "yo_NG.UTF-8", "yue_HK", "yue_HK.UTF-8", "yuw_PG", "yuw_PG.UTF-8", "zh_CN", "zh_CN.GB18030", "zh_CN.GB2312", "zh_CN.GBK", "zh_CN.UTF-8", "zh_HK", "zh_HK.BIG5-HKSCS", "zh_HK.UTF-8", "zh_SG", "zh_SG.GB2312", "zh_SG.GBK", "zh_SG.UTF-8", "zh_TW", "zh_TW.BIG5", "zh_TW.EUC-TW", "zh_TW.UTF-8", "zu_ZA", "zu_ZA.ISO-8859-1", "zu_ZA.UTF-8"}; std::string ConsumeLocaleIdentifier(FuzzedDataProvider& fuzzed_data_provider) { @@ -56,7 +55,10 @@ void test_one_input(const std::vector<uint8_t>& buffer) const int64_t atoi64c_without_locale = atoi64(random_string.c_str()); const int64_t random_int64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); const std::string tostring_without_locale = ToString(random_int64); + // The variable `random_int32` is no longer used, but the harness still needs to + // consume the same data that it did previously to not invalidate existing seeds. const int32_t random_int32 = fuzzed_data_provider.ConsumeIntegral<int32_t>(); + (void)random_int32; const std::string strprintf_int_without_locale = strprintf("%d", random_int64); const double random_double = fuzzed_data_provider.ConsumeFloatingPoint<double>(); const std::string strprintf_double_without_locale = strprintf("%f", random_double); diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index 571364aaa6..a269378607 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/prevector.cpp b/src/test/fuzz/prevector.cpp new file mode 100644 index 0000000000..64920f4af5 --- /dev/null +++ b/src/test/fuzz/prevector.cpp @@ -0,0 +1,263 @@ +// Copyright (c) 2015-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> + +#include <prevector.h> +#include <vector> + +#include <reverse_iterator.h> +#include <serialize.h> +#include <streams.h> + +namespace { + +template<unsigned int N, typename T> +class prevector_tester { + typedef std::vector<T> realtype; + realtype real_vector; + realtype real_vector_alt; + + typedef prevector<N, T> pretype; + pretype pre_vector; + pretype pre_vector_alt; + + typedef typename pretype::size_type Size; + +public: + void test() const { + const pretype& const_pre_vector = pre_vector; + assert(real_vector.size() == pre_vector.size()); + assert(real_vector.empty() == pre_vector.empty()); + for (Size s = 0; s < real_vector.size(); s++) { + assert(real_vector[s] == pre_vector[s]); + assert(&(pre_vector[s]) == &(pre_vector.begin()[s])); + assert(&(pre_vector[s]) == &*(pre_vector.begin() + s)); + assert(&(pre_vector[s]) == &*((pre_vector.end() + s) - real_vector.size())); + } + // assert(realtype(pre_vector) == real_vector); + assert(pretype(real_vector.begin(), real_vector.end()) == pre_vector); + assert(pretype(pre_vector.begin(), pre_vector.end()) == pre_vector); + size_t pos = 0; + for (const T& v : pre_vector) { + assert(v == real_vector[pos]); + ++pos; + } + for (const T& v : reverse_iterate(pre_vector)) { + --pos; + assert(v == real_vector[pos]); + } + for (const T& v : const_pre_vector) { + assert(v == real_vector[pos]); + ++pos; + } + for (const T& v : reverse_iterate(const_pre_vector)) { + --pos; + assert(v == real_vector[pos]); + } + CDataStream ss1(SER_DISK, 0); + CDataStream ss2(SER_DISK, 0); + ss1 << real_vector; + ss2 << pre_vector; + assert(ss1.size() == ss2.size()); + for (Size s = 0; s < ss1.size(); s++) { + assert(ss1[s] == ss2[s]); + } + } + + void resize(Size s) { + real_vector.resize(s); + assert(real_vector.size() == s); + pre_vector.resize(s); + assert(pre_vector.size() == s); + } + + void reserve(Size s) { + real_vector.reserve(s); + assert(real_vector.capacity() >= s); + pre_vector.reserve(s); + assert(pre_vector.capacity() >= s); + } + + void insert(Size position, const T& value) { + real_vector.insert(real_vector.begin() + position, value); + pre_vector.insert(pre_vector.begin() + position, value); + } + + void insert(Size position, Size count, const T& value) { + real_vector.insert(real_vector.begin() + position, count, value); + pre_vector.insert(pre_vector.begin() + position, count, value); + } + + template<typename I> + void insert_range(Size position, I first, I last) { + real_vector.insert(real_vector.begin() + position, first, last); + pre_vector.insert(pre_vector.begin() + position, first, last); + } + + void erase(Size position) { + real_vector.erase(real_vector.begin() + position); + pre_vector.erase(pre_vector.begin() + position); + } + + void erase(Size first, Size last) { + real_vector.erase(real_vector.begin() + first, real_vector.begin() + last); + pre_vector.erase(pre_vector.begin() + first, pre_vector.begin() + last); + } + + void update(Size pos, const T& value) { + real_vector[pos] = value; + pre_vector[pos] = value; + } + + void push_back(const T& value) { + real_vector.push_back(value); + pre_vector.push_back(value); + } + + void pop_back() { + real_vector.pop_back(); + pre_vector.pop_back(); + } + + void clear() { + real_vector.clear(); + pre_vector.clear(); + } + + void assign(Size n, const T& value) { + real_vector.assign(n, value); + pre_vector.assign(n, value); + } + + Size size() const { + return real_vector.size(); + } + + Size capacity() const { + return pre_vector.capacity(); + } + + void shrink_to_fit() { + pre_vector.shrink_to_fit(); + } + + void swap() { + real_vector.swap(real_vector_alt); + pre_vector.swap(pre_vector_alt); + } + + void move() { + real_vector = std::move(real_vector_alt); + real_vector_alt.clear(); + pre_vector = std::move(pre_vector_alt); + pre_vector_alt.clear(); + } + + void copy() { + real_vector = real_vector_alt; + pre_vector = pre_vector_alt; + } + + void resize_uninitialized(realtype values) { + size_t r = values.size(); + size_t s = real_vector.size() / 2; + if (real_vector.capacity() < s + r) { + real_vector.reserve(s + r); + } + real_vector.resize(s); + pre_vector.resize_uninitialized(s); + for (auto v : values) { + real_vector.push_back(v); + } + auto p = pre_vector.size(); + pre_vector.resize_uninitialized(p + r); + for (auto v : values) { + pre_vector[p] = v; + ++p; + } + } +}; + +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider prov(buffer.data(), buffer.size()); + prevector_tester<8, int> test; + + while (prov.remaining_bytes()) { + switch (prov.ConsumeIntegralInRange<int>(0, 13 + 3 * (test.size() > 0))) { + case 0: + test.insert(prov.ConsumeIntegralInRange<size_t>(0, test.size()), prov.ConsumeIntegral<int>()); + break; + case 1: + test.resize(std::max(0, std::min(30, (int)test.size() + prov.ConsumeIntegralInRange<int>(0, 4) - 2))); + break; + case 2: + test.insert(prov.ConsumeIntegralInRange<size_t>(0, test.size()), 1 + prov.ConsumeBool(), prov.ConsumeIntegral<int>()); + break; + case 3: { + int del = prov.ConsumeIntegralInRange<int>(0, test.size()); + int beg = prov.ConsumeIntegralInRange<int>(0, test.size() - del); + test.erase(beg, beg + del); + break; + } + case 4: + test.push_back(prov.ConsumeIntegral<int>()); + break; + case 5: { + int values[4]; + int num = 1 + prov.ConsumeIntegralInRange<int>(0, 3); + for (int k = 0; k < num; ++k) { + values[k] = prov.ConsumeIntegral<int>(); + } + test.insert_range(prov.ConsumeIntegralInRange<size_t>(0, test.size()), values, values + num); + break; + } + case 6: { + int num = 1 + prov.ConsumeIntegralInRange<int>(0, 15); + std::vector<int> values(num); + for (auto& v : values) { + v = prov.ConsumeIntegral<int>(); + } + test.resize_uninitialized(values); + break; + } + case 7: + test.reserve(prov.ConsumeIntegralInRange<size_t>(0, 32767)); + break; + case 8: + test.shrink_to_fit(); + break; + case 9: + test.clear(); + break; + case 10: + test.assign(prov.ConsumeIntegralInRange<size_t>(0, 32767), prov.ConsumeIntegral<int>()); + break; + case 11: + test.swap(); + break; + case 12: + test.copy(); + break; + case 13: + test.move(); + break; + case 14: + test.update(prov.ConsumeIntegralInRange<size_t>(0, test.size() - 1), prov.ConsumeIntegral<int>()); + break; + case 15: + test.erase(prov.ConsumeIntegralInRange<size_t>(0, test.size() - 1)); + break; + case 16: + test.pop_back(); + break; + } + } + + test.test(); +} diff --git a/src/test/fuzz/primitives_transaction.cpp b/src/test/fuzz/primitives_transaction.cpp new file mode 100644 index 0000000000..2e5ba6bdb0 --- /dev/null +++ b/src/test/fuzz/primitives_transaction.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <optional.h> +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const CScript script = ConsumeScript(fuzzed_data_provider); + const Optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (out_point) { + const CTxIn tx_in{*out_point, script, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + (void)tx_in; + } + const CTxOut tx_out_1{ConsumeMoney(fuzzed_data_provider), script}; + const CTxOut tx_out_2{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; + assert((tx_out_1 == tx_out_2) != (tx_out_1 != tx_out_2)); + const Optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const Optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (mutable_tx_1 && mutable_tx_2) { + const CTransaction tx_1{*mutable_tx_1}; + const CTransaction tx_2{*mutable_tx_2}; + assert((tx_1 == tx_2) != (tx_1 != tx_2)); + } +} diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index dc49dd499a..c03365199a 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -19,20 +19,17 @@ #include <validationinterface.h> #include <version.h> -#include <algorithm> #include <atomic> #include <cassert> #include <chrono> #include <cstdint> #include <iosfwd> #include <iostream> -#include <map> #include <memory> -#include <set> #include <string> #include <vector> -bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); +bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); namespace { @@ -44,25 +41,17 @@ const std::string LIMIT_TO_MESSAGE_TYPE{TO_STRING(MESSAGE_TYPE)}; const std::string LIMIT_TO_MESSAGE_TYPE; #endif -const std::map<std::string, std::set<std::string>> EXPECTED_DESERIALIZATION_EXCEPTIONS = { - {"CDataStream::read(): end of data: iostream error", {"addr", "block", "blocktxn", "cmpctblock", "feefilter", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "ping", "sendcmpct", "tx"}}, - {"CompactSize exceeds limit of type: iostream error", {"cmpctblock"}}, - {"differential value overflow: iostream error", {"getblocktxn"}}, - {"index overflowed 16 bits: iostream error", {"getblocktxn"}}, - {"index overflowed 16-bits: iostream error", {"cmpctblock"}}, - {"indexes overflowed 16 bits: iostream error", {"getblocktxn"}}, - {"non-canonical ReadCompactSize(): iostream error", {"addr", "block", "blocktxn", "cmpctblock", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "tx"}}, - {"ReadCompactSize(): size too large: iostream error", {"addr", "block", "blocktxn", "cmpctblock", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "tx"}}, - {"Superfluous witness record: iostream error", {"block", "blocktxn", "cmpctblock", "tx"}}, - {"Unknown transaction optional data: iostream error", {"block", "blocktxn", "cmpctblock", "tx"}}, -}; - -const RegTestingSetup* g_setup; +const TestingSetup* g_setup; } // namespace void initialize() { - static RegTestingSetup setup{}; + static TestingSetup setup{ + CBaseChainParams::REGTEST, + { + "-nodebuglogfile", + }, + }; g_setup = &setup; for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { @@ -86,13 +75,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) g_setup->m_node.peer_logic->InitializeNode(&p2p_node); try { (void)ProcessMessage(&p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), Params(), *g_setup->m_node.mempool, g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), std::atomic<bool>{false}); - } catch (const std::ios_base::failure& e) { - const std::string exception_message{e.what()}; - const auto p = EXPECTED_DESERIALIZATION_EXCEPTIONS.find(exception_message); - if (p == EXPECTED_DESERIALIZATION_EXCEPTIONS.cend() || p->second.count(random_message_type) == 0) { - std::cout << "Unexpected exception when processing message type \"" << random_message_type << "\": " << exception_message << std::endl; - assert(false); - } + } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); } diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp new file mode 100644 index 0000000000..bcbf65bdca --- /dev/null +++ b/src/test/fuzz/process_messages.cpp @@ -0,0 +1,80 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/consensus.h> +#include <net.h> +#include <net_processing.h> +#include <protocol.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/mining.h> +#include <test/util/net.h> +#include <test/util/setup_common.h> +#include <util/memory.h> +#include <validation.h> +#include <validationinterface.h> + +const TestingSetup* g_setup; + +void initialize() +{ + static TestingSetup setup{ + CBaseChainParams::REGTEST, + { + "-nodebuglogfile", + }, + }; + g_setup = &setup; + + for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { + MineBlock(g_setup->m_node, CScript() << OP_TRUE); + } + SyncWithValidationInterfaceQueue(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); + std::vector<CNode*> peers; + + const auto num_peers_to_add = fuzzed_data_provider.ConsumeIntegralInRange(1, 3); + for (int i = 0; i < num_peers_to_add; ++i) { + const ServiceFlags service_flags = ServiceFlags(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + const bool inbound{fuzzed_data_provider.ConsumeBool()}; + const bool block_relay_only{fuzzed_data_provider.ConsumeBool()}; + peers.push_back(MakeUnique<CNode>(i, service_flags, 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, inbound, block_relay_only).release()); + CNode& p2p_node = *peers.back(); + + p2p_node.fSuccessfullyConnected = true; + p2p_node.fPauseSend = false; + p2p_node.nVersion = PROTOCOL_VERSION; + p2p_node.SetSendVersion(PROTOCOL_VERSION); + g_setup->m_node.peer_logic->InitializeNode(&p2p_node); + + connman.AddTestNode(p2p_node); + } + + while (fuzzed_data_provider.ConsumeBool()) { + const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; + + CSerializedNetMsg net_msg; + net_msg.command = random_message_type; + net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + + CNode& random_node = *peers.at(fuzzed_data_provider.ConsumeIntegralInRange<int>(0, peers.size() - 1)); + + (void)connman.ReceiveMsgFrom(random_node, net_msg); + random_node.fPauseSend = false; + + try { + connman.ProcessMessagesOnce(random_node); + } catch (const std::ios_base::failure&) { + } + } + connman.ClearTestNodes(); + SyncWithValidationInterfaceQueue(); +} diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index ca3e0b8586..64328fb66e 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 80e2f234d7..de82122dd6 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 3d8ece7c61..ffc65eedc0 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/script_ops.cpp b/src/test/fuzz/script_ops.cpp index 0cd129ba7a..7d24af20ac 100644 --- a/src/test/fuzz/script_ops.cpp +++ b/src/test/fuzz/script_ops.cpp @@ -17,12 +17,16 @@ void test_one_input(const std::vector<uint8_t>& buffer) CScript script = ConsumeScript(fuzzed_data_provider); while (fuzzed_data_provider.remaining_bytes() > 0) { switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 7)) { - case 0: - script += ConsumeScript(fuzzed_data_provider); + case 0: { + CScript s = ConsumeScript(fuzzed_data_provider); + script = std::move(s); break; - case 1: - script = script + ConsumeScript(fuzzed_data_provider); + } + case 1: { + const CScript& s = ConsumeScript(fuzzed_data_provider); + script = s; break; + } case 2: script << fuzzed_data_provider.ConsumeIntegral<int64_t>(); break; diff --git a/src/test/fuzz/signature_checker.cpp b/src/test/fuzz/signature_checker.cpp index 312db27adc..4a8c7a63af 100644 --- a/src/test/fuzz/signature_checker.cpp +++ b/src/test/fuzz/signature_checker.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index d8e84f1a0f..d6deb7fc3d 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index b70ea6d90e..9c7b0d47a2 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -120,4 +120,15 @@ NODISCARD bool MultiplicationOverflow(const T i, const T j) noexcept } } +template <class T> +NODISCARD bool AdditionOverflow(const T i, const T j) noexcept +{ + static_assert(std::is_integral<T>::value, "Integral required."); + if (std::numeric_limits<T>::is_signed) { + return (i > 0 && j > std::numeric_limits<T>::max() - i) || + (i < 0 && j < std::numeric_limits<T>::min() - i); + } + return std::numeric_limits<T>::max() - i < j; +} + #endif // BITCOIN_TEST_FUZZ_UTIL_H diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 10fb05ca8a..512e48f8e5 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <test/util/setup_common.h> #include <util/strencodings.h> #include <util/system.h> -#include <test/util/setup_common.h> #include <string> #include <utility> diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index b864e6e599..87f6470afa 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2013-2019 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <clientversion.h> #include <crypto/siphash.h> #include <hash.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/interfaces_tests.cpp b/src/test/interfaces_tests.cpp new file mode 100644 index 0000000000..b0d4de89f3 --- /dev/null +++ b/src/test/interfaces_tests.cpp @@ -0,0 +1,163 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <consensus/validation.h> +#include <interfaces/chain.h> +#include <script/standard.h> +#include <test/util/setup_common.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> + +using interfaces::FoundBlock; + +BOOST_FIXTURE_TEST_SUITE(interfaces_tests, TestChain100Setup) + +BOOST_AUTO_TEST_CASE(findBlock) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + + uint256 hash; + BOOST_CHECK(chain->findBlock(active[10]->GetBlockHash(), FoundBlock().hash(hash))); + BOOST_CHECK_EQUAL(hash, active[10]->GetBlockHash()); + + int height = -1; + BOOST_CHECK(chain->findBlock(active[20]->GetBlockHash(), FoundBlock().height(height))); + BOOST_CHECK_EQUAL(height, active[20]->nHeight); + + CBlock data; + BOOST_CHECK(chain->findBlock(active[30]->GetBlockHash(), FoundBlock().data(data))); + BOOST_CHECK_EQUAL(data.GetHash(), active[30]->GetBlockHash()); + + int64_t time = -1; + BOOST_CHECK(chain->findBlock(active[40]->GetBlockHash(), FoundBlock().time(time))); + BOOST_CHECK_EQUAL(time, active[40]->GetBlockTime()); + + int64_t max_time = -1; + BOOST_CHECK(chain->findBlock(active[50]->GetBlockHash(), FoundBlock().maxTime(max_time))); + BOOST_CHECK_EQUAL(max_time, active[50]->GetBlockTimeMax()); + + int64_t mtp_time = -1; + BOOST_CHECK(chain->findBlock(active[60]->GetBlockHash(), FoundBlock().mtpTime(mtp_time))); + BOOST_CHECK_EQUAL(mtp_time, active[60]->GetMedianTimePast()); + + BOOST_CHECK(!chain->findBlock({}, FoundBlock())); +} + +BOOST_AUTO_TEST_CASE(findFirstBlockWithTimeAndHeight) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + uint256 hash; + int height; + BOOST_CHECK(chain->findFirstBlockWithTimeAndHeight(/* min_time= */ 0, /* min_height= */ 5, FoundBlock().hash(hash).height(height))); + BOOST_CHECK_EQUAL(hash, active[5]->GetBlockHash()); + BOOST_CHECK_EQUAL(height, 5); + BOOST_CHECK(!chain->findFirstBlockWithTimeAndHeight(/* min_time= */ active.Tip()->GetBlockTimeMax() + 1, /* min_height= */ 0)); +} + +BOOST_AUTO_TEST_CASE(findNextBlock) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + bool reorg; + uint256 hash; + BOOST_CHECK(chain->findNextBlock(active[20]->GetBlockHash(), 20, FoundBlock().hash(hash), &reorg)); + BOOST_CHECK_EQUAL(hash, active[21]->GetBlockHash()); + BOOST_CHECK_EQUAL(reorg, false); + BOOST_CHECK(!chain->findNextBlock(uint256(), 20, {}, &reorg)); + BOOST_CHECK_EQUAL(reorg, true); + BOOST_CHECK(!chain->findNextBlock(active.Tip()->GetBlockHash(), active.Height(), {}, &reorg)); + BOOST_CHECK_EQUAL(reorg, false); +} + +BOOST_AUTO_TEST_CASE(findAncestorByHeight) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + uint256 hash; + BOOST_CHECK(chain->findAncestorByHeight(active[20]->GetBlockHash(), 10, FoundBlock().hash(hash))); + BOOST_CHECK_EQUAL(hash, active[10]->GetBlockHash()); + BOOST_CHECK(!chain->findAncestorByHeight(active[10]->GetBlockHash(), 20)); +} + +BOOST_AUTO_TEST_CASE(findAncestorByHash) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + int height = -1; + BOOST_CHECK(chain->findAncestorByHash(active[20]->GetBlockHash(), active[10]->GetBlockHash(), FoundBlock().height(height))); + BOOST_CHECK_EQUAL(height, 10); + BOOST_CHECK(!chain->findAncestorByHash(active[10]->GetBlockHash(), active[20]->GetBlockHash())); +} + +BOOST_AUTO_TEST_CASE(findCommonAncestor) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + auto* orig_tip = active.Tip(); + for (int i = 0; i < 10; ++i) { + BlockValidationState state; + ChainstateActive().InvalidateBlock(state, Params(), active.Tip()); + } + BOOST_CHECK_EQUAL(active.Height(), orig_tip->nHeight - 10); + coinbaseKey.MakeNewKey(true); + for (int i = 0; i < 20; ++i) { + CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + } + BOOST_CHECK_EQUAL(active.Height(), orig_tip->nHeight + 10); + uint256 fork_hash; + int fork_height; + int orig_height; + BOOST_CHECK(chain->findCommonAncestor(orig_tip->GetBlockHash(), active.Tip()->GetBlockHash(), FoundBlock().height(fork_height).hash(fork_hash), FoundBlock().height(orig_height))); + BOOST_CHECK_EQUAL(orig_height, orig_tip->nHeight); + BOOST_CHECK_EQUAL(fork_height, orig_tip->nHeight - 10); + BOOST_CHECK_EQUAL(fork_hash, active[fork_height]->GetBlockHash()); + + uint256 active_hash, orig_hash; + BOOST_CHECK(!chain->findCommonAncestor(active.Tip()->GetBlockHash(), {}, {}, FoundBlock().hash(active_hash), {})); + BOOST_CHECK(!chain->findCommonAncestor({}, orig_tip->GetBlockHash(), {}, {}, FoundBlock().hash(orig_hash))); + BOOST_CHECK_EQUAL(active_hash, active.Tip()->GetBlockHash()); + BOOST_CHECK_EQUAL(orig_hash, orig_tip->GetBlockHash()); +} + +BOOST_AUTO_TEST_CASE(hasBlocks) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + + // Test ranges + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + active[5]->nStatus &= ~BLOCK_HAVE_DATA; + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + active[95]->nStatus &= ~BLOCK_HAVE_DATA; + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + active[50]->nStatus &= ~BLOCK_HAVE_DATA; + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + + // Test edge cases + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 6, 49)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 5, 49)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 6, 50)); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index b52513f4af..d465ee6759 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <key.h> #include <key_io.h> #include <script/script.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 034b7938f9..cf2bd03698 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> #include <key_io.h> +#include <test/util/setup_common.h> #include <uint256.h> -#include <util/system.h> #include <util/strencodings.h> #include <util/string.h> -#include <test/util/setup_common.h> +#include <util/system.h> #include <string> #include <vector> diff --git a/src/test/main.cpp b/src/test/main.cpp index f32243d1d3..5885564074 100644 --- a/src/test/main.cpp +++ b/src/test/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/merkleblock_tests.cpp b/src/test/merkleblock_tests.cpp index 9f8c4ba5c5..98b27994a6 100644 --- a/src/test/merkleblock_tests.cpp +++ b/src/test/merkleblock_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <merkleblock.h> -#include <uint256.h> #include <test/util/setup_common.h> +#include <uint256.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index 97a918da45..dd2890c134 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> #include <policy/policy.h> +#include <script/interpreter.h> #include <script/script.h> #include <script/script_error.h> -#include <script/interpreter.h> #include <script/sign.h> #include <script/signingprovider.h> +#include <test/util/setup_common.h> #include <tinyformat.h> #include <uint256.h> -#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 9b5a86fef2..84bf593497 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -1,23 +1,24 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrdb.h> #include <addrman.h> +#include <chainparams.h> #include <clientversion.h> -#include <test/util/setup_common.h> -#include <string> -#include <boost/test/unit_test.hpp> -#include <serialize.h> -#include <streams.h> #include <net.h> #include <netbase.h> -#include <chainparams.h> +#include <serialize.h> +#include <streams.h> +#include <test/util/setup_common.h> #include <util/memory.h> -#include <util/system.h> #include <util/string.h> +#include <util/system.h> + +#include <boost/test/unit_test.hpp> #include <memory> +#include <string> class CAddrManSerializationMock : public CAddrMan { diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 9730b40580..ec6a290334 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <netbase.h> #include <net_permissions.h> +#include <netbase.h> #include <test/util/setup_common.h> #include <util/strencodings.h> diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index bf58bd63b9..a9d661438c 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <arith_uint256.h> #include <consensus/merkle.h> #include <merkleblock.h> #include <serialize.h> #include <streams.h> +#include <test/util/setup_common.h> #include <uint256.h> -#include <arith_uint256.h> #include <version.h> -#include <test/util/setup_common.h> #include <vector> diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 025e2b78ca..06877898a4 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <policy/policy.h> #include <policy/fees.h> +#include <policy/policy.h> #include <txmempool.h> #include <uint256.h> #include <util/time.h> diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 9782b78f2c..12c5848eaf 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <vector> #include <prevector.h> +#include <vector> #include <reverse_iterator.h> #include <serialize.h> diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index e0df41a971..ca3b92f2e1 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <boost/test/unit_test.hpp> -#include <random> #include <algorithm> +#include <random> BOOST_FIXTURE_TEST_SUITE(random_tests, BasicTestingSetup) diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp index 4e51b8c02a..a42608a66d 100644 --- a/src/test/reverselock_tests.cpp +++ b/src/test/reverselock_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 84a3980b19..d9c66f1c19 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <rpc/server.h> #include <rpc/client.h> +#include <rpc/server.h> #include <rpc/util.h> #include <core_io.h> diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 801cf8e5d1..1395a7f38c 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,15 +6,17 @@ #include <scheduler.h> #include <util/time.h> -#include <boost/thread.hpp> #include <boost/test/unit_test.hpp> +#include <boost/thread.hpp> + +#include <mutex> BOOST_AUTO_TEST_SUITE(scheduler_tests) -static void microTask(CScheduler& s, boost::mutex& mutex, int& counter, int delta, std::chrono::system_clock::time_point rescheduleTime) +static void microTask(CScheduler& s, std::mutex& mutex, int& counter, int delta, std::chrono::system_clock::time_point rescheduleTime) { { - boost::unique_lock<boost::mutex> lock(mutex); + std::lock_guard<std::mutex> lock(mutex); counter += delta; } std::chrono::system_clock::time_point noTime = std::chrono::system_clock::time_point::min(); @@ -38,7 +40,7 @@ BOOST_AUTO_TEST_CASE(manythreads) // counters should sum to the number of initial tasks performed. CScheduler microTasks; - boost::mutex counterMutex[10]; + std::mutex counterMutex[10]; int counter[10] = { 0 }; FastRandomContext rng{/* fDeterministic */ true}; auto zeroToNine = [](FastRandomContext& rc) -> int { return rc.randrange(10); }; // [0, 9] diff --git a/src/test/script_p2sh_tests.cpp b/src/test/script_p2sh_tests.cpp index 8c1e843b0b..f6824a4e5e 100644 --- a/src/test/script_p2sh_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/tx_verify.h> #include <key.h> -#include <validation.h> #include <policy/policy.h> +#include <policy/settings.h> #include <script/script.h> #include <script/script_error.h> -#include <policy/settings.h> #include <script/sign.h> #include <script/signingprovider.h> #include <test/util/setup_common.h> +#include <validation.h> #include <vector> diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 26015ca4c2..56454f61f3 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,16 +6,16 @@ #include <core_io.h> #include <key.h> +#include <rpc/util.h> #include <script/script.h> #include <script/script_error.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <util/system.h> -#include <util/strencodings.h> -#include <test/util/transaction_utils.h> -#include <test/util/setup_common.h> -#include <rpc/util.h> #include <streams.h> +#include <test/util/setup_common.h> +#include <test/util/transaction_utils.h> +#include <util/strencodings.h> +#include <util/system.h> #if defined(HAVE_CONSENSUS_LIB) #include <script/bitcoinconsensus.h> @@ -217,7 +217,6 @@ struct KeyData KeyData() { - key0.Set(vchKey0, vchKey0 + 32, false); key0C.Set(vchKey0, vchKey0 + 32, true); pubkey0 = key0.GetPubKey(); @@ -272,9 +271,9 @@ private: void DoPush(const std::vector<unsigned char>& data) { - DoPush(); - push = data; - havePush = true; + DoPush(); + push = data; + havePush = true; } public: @@ -306,10 +305,10 @@ public: return *this; } - TestBuilder& Add(const CScript& _script) + TestBuilder& Opcode(const opcodetype& _op) { DoPush(); - spendTx.vin[0].scriptSig += _script; + spendTx.vin[0].scriptSig << _op; return *this; } @@ -326,8 +325,9 @@ public: return *this; } - TestBuilder& Push(const CScript& _script) { - DoPush(std::vector<unsigned char>(_script.begin(), _script.end())); + TestBuilder& Push(const CScript& _script) + { + DoPush(std::vector<unsigned char>(_script.begin(), _script.end())); return *this; } @@ -681,22 +681,22 @@ BOOST_AUTO_TEST_CASE(script_build) tests.push_back(TestBuilder(CScript() << OP_2 << ToByteVector(keys.pubkey1C) << ToByteVector(keys.pubkey1C) << OP_2 << OP_CHECKMULTISIG, "2-of-2 with two identical keys and sigs pushed using OP_DUP but no SIGPUSHONLY", 0 - ).Num(0).PushSig(keys.key1).Add(CScript() << OP_DUP)); + ).Num(0).PushSig(keys.key1).Opcode(OP_DUP)); tests.push_back(TestBuilder(CScript() << OP_2 << ToByteVector(keys.pubkey1C) << ToByteVector(keys.pubkey1C) << OP_2 << OP_CHECKMULTISIG, "2-of-2 with two identical keys and sigs pushed using OP_DUP", SCRIPT_VERIFY_SIGPUSHONLY - ).Num(0).PushSig(keys.key1).Add(CScript() << OP_DUP).ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); + ).Num(0).PushSig(keys.key1).Opcode(OP_DUP).ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); tests.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey2C) << OP_CHECKSIG, "P2SH(P2PK) with non-push scriptSig but no P2SH or SIGPUSHONLY", 0, true - ).PushSig(keys.key2).Add(CScript() << OP_NOP8).PushRedeem()); + ).PushSig(keys.key2).Opcode(OP_NOP8).PushRedeem()); tests.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey2C) << OP_CHECKSIG, "P2PK with non-push scriptSig but with P2SH validation", 0 - ).PushSig(keys.key2).Add(CScript() << OP_NOP8)); + ).PushSig(keys.key2).Opcode(OP_NOP8)); tests.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey2C) << OP_CHECKSIG, "P2SH(P2PK) with non-push scriptSig but no SIGPUSHONLY", SCRIPT_VERIFY_P2SH, true - ).PushSig(keys.key2).Add(CScript() << OP_NOP8).PushRedeem().ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); + ).PushSig(keys.key2).Opcode(OP_NOP8).PushRedeem().ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); tests.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey2C) << OP_CHECKSIG, "P2SH(P2PK) with non-push scriptSig but not P2SH", SCRIPT_VERIFY_SIGPUSHONLY, true - ).PushSig(keys.key2).Add(CScript() << OP_NOP8).PushRedeem().ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); + ).PushSig(keys.key2).Opcode(OP_NOP8).PushRedeem().ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); tests.push_back(TestBuilder(CScript() << OP_2 << ToByteVector(keys.pubkey1C) << ToByteVector(keys.pubkey1C) << OP_2 << OP_CHECKMULTISIG, "2-of-2 with two identical keys and sigs pushed", SCRIPT_VERIFY_SIGPUSHONLY ).Num(0).PushSig(keys.key1).PushSig(keys.key1)); @@ -1470,24 +1470,6 @@ BOOST_AUTO_TEST_CASE(script_HasValidOps) BOOST_CHECK(!script.HasValidOps()); } -BOOST_AUTO_TEST_CASE(script_can_append_self) -{ - CScript s, d; - - s = ScriptFromHex("00"); - s += s; - d = ScriptFromHex("0000"); - BOOST_CHECK(s == d); - - // check doubling a script that's large enough to require reallocation - static const char hex[] = "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"; - s = CScript() << ParseHex(hex) << OP_CHECKSIG; - d = CScript() << ParseHex(hex) << OP_CHECKSIG << ParseHex(hex) << OP_CHECKSIG; - s += s; - BOOST_CHECK(s == d); -} - - #if defined(HAVE_CONSENSUS_LIB) /* Test simple (successful) usage of bitcoinconsensus_verify_script */ diff --git a/src/test/scriptnum10.h b/src/test/scriptnum10.h index 9f928827cb..352797f18d 100644 --- a/src/test/scriptnum10.h +++ b/src/test/scriptnum10.h @@ -1,17 +1,17 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TEST_SCRIPTNUM10_H #define BITCOIN_TEST_SCRIPTNUM10_H +#include <assert.h> #include <limits> #include <stdexcept> #include <stdint.h> #include <string> #include <vector> -#include <assert.h> class scriptnum10_error : public std::runtime_error { diff --git a/src/test/scriptnum_tests.cpp b/src/test/scriptnum_tests.cpp index 40a6f69668..281018be9f 100644 --- a/src/test/scriptnum_tests.cpp +++ b/src/test/scriptnum_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/scriptnum10.h> #include <script/script.h> +#include <test/scriptnum10.h> #include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index ea600499ca..9a6c721ab8 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <hash.h> #include <serialize.h> #include <streams.h> -#include <hash.h> #include <test/util/setup_common.h> #include <util/strencodings.h> diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 10b161aa80..fcd831ccda 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index bcc4a46873..5ca136ea6e 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -1,18 +1,18 @@ -// Copyright (c) 2013-2019 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/tx_check.h> #include <consensus/validation.h> -#include <test/data/sighash.json.h> #include <hash.h> #include <script/interpreter.h> #include <script/script.h> #include <serialize.h> #include <streams.h> +#include <test/data/sighash.json.h> #include <test/util/setup_common.h> -#include <util/system.h> #include <util/strencodings.h> +#include <util/system.h> #include <version.h> #include <iostream> diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 6462fcefe3..6e36bce7a1 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/consensus.h> #include <consensus/tx_verify.h> -#include <pubkey.h> #include <key.h> +#include <pubkey.h> #include <script/script.h> #include <script/standard.h> -#include <uint256.h> #include <test/util/setup_common.h> +#include <uint256.h> #include <vector> diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp index 29b43e9bec..8c880babd1 100644 --- a/src/test/timedata_tests.cpp +++ b/src/test/timedata_tests.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include <netaddress.h> #include <noui.h> -#include <util/string.h> #include <test/util/logging.h> #include <test/util/setup_common.h> #include <timedata.h> +#include <util/string.h> #include <warnings.h> #include <string> diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 96520079d7..9d0ae56c3f 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,23 +6,23 @@ #include <test/data/tx_valid.json.h> #include <test/util/setup_common.h> -#include <clientversion.h> #include <checkqueue.h> +#include <clientversion.h> #include <consensus/tx_check.h> #include <consensus/validation.h> #include <core_io.h> #include <key.h> -#include <validation.h> #include <policy/policy.h> #include <policy/settings.h> #include <script/script.h> +#include <script/script_error.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/script_error.h> #include <script/standard.h> #include <streams.h> -#include <util/strencodings.h> #include <test/util/transaction_utils.h> +#include <util/strencodings.h> +#include <validation.h> #include <map> #include <string> diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 3550a02316..5fc172ee86 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -70,12 +70,8 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) // shutdown sequence (c.f. Shutdown() in init.cpp) txindex.Stop(); - // txindex job may be scheduled, so stop scheduler before destructing - m_node.scheduler->stop(); - threadGroup.interrupt_all(); - threadGroup.join_all(); - - // Rest of shutdown sequence and destructors happen in ~TestingSetup() + // Let scheduler events finish running to avoid accessing any memory related to txindex after it is destructed + SyncWithValidationInterfaceQueue(); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index cace75f093..c3d7af8323 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <validation.h> #include <consensus/validation.h> #include <primitives/transaction.h> #include <script/script.h> #include <test/util/setup_common.h> +#include <validation.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 7842594b80..cdef7dcc3c 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -4,12 +4,12 @@ #include <consensus/validation.h> #include <key.h> -#include <validation.h> -#include <txmempool.h> -#include <script/standard.h> #include <script/sign.h> #include <script/signingprovider.h> +#include <script/standard.h> #include <test/util/setup_common.h> +#include <txmempool.h> +#include <validation.h> #include <boost/test/unit_test.hpp> @@ -108,7 +108,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // any script flag that is implemented as an upgraded NOP code. static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - PrecomputedTransactionData txdata(tx); + PrecomputedTransactionData txdata; // If we add many more flags, this loop can get too expensive, but we can // rewrite in the future to randomly pick a set of flags to evaluate. for (uint32_t test_flags=0; test_flags < (1U << 16); test_flags += 1) { @@ -200,7 +200,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) LOCK(cs_main); TxValidationState state; - PrecomputedTransactionData ptd_spend_tx(spend_tx); + PrecomputedTransactionData ptd_spend_tx; BOOST_CHECK(!CheckInputScripts(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); @@ -269,7 +269,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 100; TxValidationState state; - PrecomputedTransactionData txdata(invalid_with_cltv_tx); + PrecomputedTransactionData txdata; BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_cltv_tx), state, ::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); } @@ -297,7 +297,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 100; TxValidationState state; - PrecomputedTransactionData txdata(invalid_with_csv_tx); + PrecomputedTransactionData txdata; BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_csv_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); } @@ -358,7 +358,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) tx.vin[1].scriptWitness.SetNull(); TxValidationState state; - PrecomputedTransactionData txdata(tx); + PrecomputedTransactionData txdata; // This transaction is now invalid under segwit, because of the second input. BOOST_CHECK(!CheckInputScripts(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 7293ecd325..c0ae2f8cf2 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -1,16 +1,17 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <arith_uint256.h> #include <streams.h> +#include <test/util/setup_common.h> #include <uint256.h> #include <version.h> -#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> -#include <sstream> + #include <iomanip> +#include <sstream> #include <string> BOOST_FIXTURE_TEST_SUITE(uint256_tests, BasicTestingSetup) diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp new file mode 100644 index 0000000000..09f2f1807f --- /dev/null +++ b/src/test/util/net.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/net.h> + +#include <chainparams.h> +#include <net.h> + +void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, const char* pch, unsigned int nBytes, bool& complete) const +{ + assert(node.ReceiveMsgBytes(pch, nBytes, complete)); + if (complete) { + size_t nSizeAdded = 0; + auto it(node.vRecvMsg.begin()); + for (; it != node.vRecvMsg.end(); ++it) { + // vRecvMsg contains only completed CNetMessage + // the single possible partially deserialized message are held by TransportDeserializer + nSizeAdded += it->m_raw_message_size; + } + { + LOCK(node.cs_vProcessMsg); + node.vProcessMsg.splice(node.vProcessMsg.end(), node.vRecvMsg, node.vRecvMsg.begin(), it); + node.nProcessQueueSize += nSizeAdded; + node.fPauseRecv = node.nProcessQueueSize > nReceiveFloodSize; + } + } +} + +bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) const +{ + std::vector<unsigned char> ser_msg_header; + node.m_serializer->prepareForTransport(ser_msg, ser_msg_header); + + bool complete; + NodeReceiveMsgBytes(node, (const char*)ser_msg_header.data(), ser_msg_header.size(), complete); + NodeReceiveMsgBytes(node, (const char*)ser_msg.data.data(), ser_msg.data.size(), complete); + return complete; +} diff --git a/src/test/util/net.h b/src/test/util/net.h new file mode 100644 index 0000000000..ca8cb7fad5 --- /dev/null +++ b/src/test/util/net.h @@ -0,0 +1,33 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_NET_H +#define BITCOIN_TEST_UTIL_NET_H + +#include <net.h> + +struct ConnmanTestMsg : public CConnman { + using CConnman::CConnman; + void AddTestNode(CNode& node) + { + LOCK(cs_vNodes); + vNodes.push_back(&node); + } + void ClearTestNodes() + { + LOCK(cs_vNodes); + for (CNode* node : vNodes) { + delete node; + } + vNodes.clear(); + } + + void ProcessMessagesOnce(CNode& node) { m_msgproc->ProcessMessages(&node, flagInterruptMsgProc); } + + void NodeReceiveMsgBytes(CNode& node, const char* pch, unsigned int nBytes, bool& complete) const; + + bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) const; +}; + +#endif // BITCOIN_TEST_UTIL_NET_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index d684b97787..bf0afc4171 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -27,12 +27,15 @@ #include <util/string.h> #include <util/time.h> #include <util/translation.h> +#include <util/url.h> +#include <util/vector.h> #include <validation.h> #include <validationinterface.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = nullptr; FastRandomContext g_insecure_rand_ctx; /** Random context to get unique temp data dirs. Separate from g_insecure_rand_ctx, which can be seeded from a const env var */ @@ -63,17 +66,34 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) return os; } -BasicTestingSetup::BasicTestingSetup(const std::string& chainName) +BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) : m_path_root{fs::temp_directory_path() / "test_common_" PACKAGE_NAME / g_insecure_rand_ctx_temp_path.rand256().ToString()} { + const std::vector<const char*> arguments = Cat( + { + "dummy", + "-printtoconsole=0", + "-logtimemicros", + "-debug", + "-debugexclude=libevent", + "-debugexclude=leveldb", + }, + extra_args); fs::create_directories(m_path_root); gArgs.ForceSetArg("-datadir", m_path_root.string()); ClearDatadirCache(); + { + SetupServerArgs(m_node); + std::string error; + const bool success{m_node.args->ParseParameters(arguments.size(), arguments.data(), error)}; + assert(success); + assert(error.empty()); + } SelectParams(chainName); SeedInsecureRand(); - gArgs.ForceSetArg("-printtoconsole", "0"); if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN); InitLogging(); + AppInitParameterInteraction(); LogInstance().StartLogging(); SHA256AutoDetect(); ECC_Start(); @@ -93,10 +113,12 @@ BasicTestingSetup::~BasicTestingSetup() { LogInstance().DisconnectTestLogger(); fs::remove_all(m_path_root); + gArgs.ClearArgs(); ECC_Stop(); } -TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) +TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) + : BasicTestingSetup(chainName, extra_args) { const CChainParams& chainparams = Params(); // Ideally we'd move all the RPC tests to the functional testing framework @@ -112,7 +134,8 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha GetMainSignals().RegisterBackgroundSignalScheduler(*g_rpc_node->scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - g_chainstate = MakeUnique<CChainState>(); + + g_chainman.InitializeChainstate(); ::ChainstateActive().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!::ChainstateActive().CanFlushToDisk()); @@ -139,6 +162,11 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. m_node.peer_logic = MakeUnique<PeerLogicValidation>(m_node.connman.get(), m_node.banman.get(), *m_node.scheduler, *m_node.mempool); + { + CConnman::Options options; + options.m_msgproc = m_node.peer_logic.get(); + m_node.connman->Init(options); + } } TestingSetup::~TestingSetup() @@ -151,10 +179,11 @@ TestingSetup::~TestingSetup() g_rpc_node = nullptr; m_node.connman.reset(); m_node.banman.reset(); + m_node.args = nullptr; m_node.mempool = nullptr; m_node.scheduler.reset(); UnloadBlockIndex(); - g_chainstate.reset(); + g_chainman.Reset(); pblocktree.reset(); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 0930309c3a..2477f9ad06 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -73,9 +73,11 @@ static constexpr CAmount CENT{1000000}; */ struct BasicTestingSetup { ECCVerifyHandle globalVerifyHandle; + NodeContext m_node; - explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~BasicTestingSetup(); + private: const fs::path m_path_root; }; @@ -84,10 +86,9 @@ private: * Included are coins database, script check threads setup. */ struct TestingSetup : public BasicTestingSetup { - NodeContext m_node; boost::thread_group threadGroup; - explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~TestingSetup(); }; diff --git a/src/test/util/transaction_utils.cpp b/src/test/util/transaction_utils.cpp index 999b803a8d..7e5bb30a2c 100644 --- a/src/test/util/transaction_utils.cpp +++ b/src/test/util/transaction_utils.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/util/transaction_utils.h> #include <coins.h> #include <script/signingprovider.h> +#include <test/util/transaction_utils.h> CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int nValue) { diff --git a/src/test/util/transaction_utils.h b/src/test/util/transaction_utils.h index f843928a5f..1beddd334b 100644 --- a/src/test/util/transaction_utils.h +++ b/src/test/util/transaction_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 73b37f909f..45b7fd4932 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -6,7 +6,7 @@ #include <clientversion.h> #include <hash.h> // For Hash() -#include <key.h> // For CKey +#include <key.h> // For CKey #include <optional.h> #include <sync.h> #include <test/util/setup_common.h> @@ -14,10 +14,10 @@ #include <uint256.h> #include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> +#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> -#include <util/spanparsing.h> #include <util/vector.h> #include <array> diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index cee4e0ce3c..f226caf717 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <test/util/setup_common.h> #include <util/string.h> #include <util/threadnames.h> -#include <test/util/setup_common.h> +#include <mutex> +#include <set> #include <thread> #include <vector> -#include <set> -#include <mutex> #if defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index afb3db36a2..b8af869b8e 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp new file mode 100644 index 0000000000..0d149285ad --- /dev/null +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -0,0 +1,107 @@ +// Copyright (c) 2019-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include <chainparams.h> +#include <consensus/validation.h> +#include <random.h> +#include <sync.h> +#include <test/util/setup_common.h> +#include <uint256.h> +#include <validation.h> +#include <validationinterface.h> + +#include <vector> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, TestingSetup) + +//! Basic tests for ChainstateManager. +//! +//! First create a legacy (IBD) chainstate, then create a snapshot chainstate. +BOOST_AUTO_TEST_CASE(chainstatemanager) +{ + ChainstateManager manager; + std::vector<CChainState*> chainstates; + const CChainParams& chainparams = Params(); + + // Create a legacy (IBD) chainstate. + // + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c1 = manager.InitializeChainstate(); + LEAVE_CRITICAL_SECTION(cs_main); + chainstates.push_back(&c1); + c1.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + WITH_LOCK(::cs_main, c1.InitCoinsCache()); + + BOOST_CHECK(!manager.IsSnapshotActive()); + BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(!manager.IsBackgroundIBD(&c1)); + auto all = manager.GetAll(); + BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); + + auto& active_chain = manager.ActiveChain(); + BOOST_CHECK_EQUAL(&active_chain, &c1.m_chain); + + BOOST_CHECK_EQUAL(manager.ActiveHeight(), -1); + + auto active_tip = manager.ActiveTip(); + auto exp_tip = c1.m_chain.Tip(); + BOOST_CHECK_EQUAL(active_tip, exp_tip); + + auto& validated_cs = manager.ValidatedChainstate(); + BOOST_CHECK_EQUAL(&validated_cs, &c1); + + // Create a snapshot-based chainstate. + // + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c2 = manager.InitializeChainstate(GetRandHash()); + LEAVE_CRITICAL_SECTION(cs_main); + chainstates.push_back(&c2); + c2.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + WITH_LOCK(::cs_main, c2.InitCoinsCache()); + // Unlike c1, which doesn't have any blocks. Gets us different tip, height. + c2.LoadGenesisBlock(chainparams); + BlockValidationState _; + BOOST_CHECK(c2.ActivateBestChain(_, chainparams, nullptr)); + + BOOST_CHECK(manager.IsSnapshotActive()); + BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(manager.IsBackgroundIBD(&c1)); + BOOST_CHECK(!manager.IsBackgroundIBD(&c2)); + auto all2 = manager.GetAll(); + BOOST_CHECK_EQUAL_COLLECTIONS(all2.begin(), all2.end(), chainstates.begin(), chainstates.end()); + + auto& active_chain2 = manager.ActiveChain(); + BOOST_CHECK_EQUAL(&active_chain2, &c2.m_chain); + + BOOST_CHECK_EQUAL(manager.ActiveHeight(), 0); + + auto active_tip2 = manager.ActiveTip(); + auto exp_tip2 = c2.m_chain.Tip(); + BOOST_CHECK_EQUAL(active_tip2, exp_tip2); + + // Ensure that these pointers actually correspond to different + // CCoinsViewCache instances. + BOOST_CHECK(exp_tip != exp_tip2); + + auto& validated_cs2 = manager.ValidatedChainstate(); + BOOST_CHECK_EQUAL(&validated_cs2, &c1); + + auto& validated_chain = manager.ValidatedChain(); + BOOST_CHECK_EQUAL(&validated_chain, &c1.m_chain); + + auto validated_tip = manager.ValidatedTip(); + exp_tip = c1.m_chain.Tip(); + BOOST_CHECK_EQUAL(validated_tip, exp_tip); + + // Let scheduler events finish running to avoid accessing memory that is going to be unloaded + SyncWithValidationInterfaceQueue(); + + WITH_LOCK(::cs_main, manager.Unload()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index c24164528f..388a2dbd13 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // diff --git a/src/test/validationinterface_tests.cpp b/src/test/validationinterface_tests.cpp new file mode 100644 index 0000000000..208be92852 --- /dev/null +++ b/src/test/validationinterface_tests.cpp @@ -0,0 +1,60 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <boost/test/unit_test.hpp> +#include <consensus/validation.h> +#include <primitives/block.h> +#include <scheduler.h> +#include <test/util/setup_common.h> +#include <util/check.h> +#include <validationinterface.h> + +BOOST_FIXTURE_TEST_SUITE(validationinterface_tests, TestingSetup) + +class TestInterface : public CValidationInterface +{ +public: + TestInterface(std::function<void()> on_call = nullptr, std::function<void()> on_destroy = nullptr) + : m_on_call(std::move(on_call)), m_on_destroy(std::move(on_destroy)) + { + } + virtual ~TestInterface() + { + if (m_on_destroy) m_on_destroy(); + } + void BlockChecked(const CBlock& block, const BlockValidationState& state) override + { + if (m_on_call) m_on_call(); + } + static void Call() + { + CBlock block; + BlockValidationState state; + GetMainSignals().BlockChecked(block, state); + } + std::function<void()> m_on_call; + std::function<void()> m_on_destroy; +}; + +// Regression test to ensure UnregisterAllValidationInterfaces calls don't +// destroy a validation interface while it is being called. Bug: +// https://github.com/bitcoin/bitcoin/pull/18551 +BOOST_AUTO_TEST_CASE(unregister_all_during_call) +{ + bool destroyed = false; + RegisterSharedValidationInterface(std::make_shared<TestInterface>( + [&] { + // First call should decrements reference count 2 -> 1 + UnregisterAllValidationInterfaces(); + BOOST_CHECK(!destroyed); + // Second call should not decrement reference count 1 -> 0 + UnregisterAllValidationInterfaces(); + BOOST_CHECK(!destroyed); + }, + [&] { destroyed = true; })); + TestInterface::Call(); + BOOST_CHECK(destroyed); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 7b59d539a6..11c6bdad91 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> -#include <versionbits.h> -#include <test/util/setup_common.h> #include <chainparams.h> -#include <validation.h> #include <consensus/params.h> +#include <test/util/setup_common.h> +#include <validation.h> +#include <versionbits.h> #include <boost/test/unit_test.hpp> diff --git a/src/txdb.cpp b/src/txdb.cpp index acc47ab45e..071aa1336b 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 47b0d39ea4..102e50dc5c 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/undo.h b/src/undo.h index 80bbd146f3..a98f046735 100644 --- a/src/undo.h +++ b/src/undo.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index 60bd27bf90..5354cdb962 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/bip32.cpp b/src/util/bip32.cpp index 6f176dd5ec..4c7e948368 100644 --- a/src/util/bip32.cpp +++ b/src/util/bip32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/bip32.h b/src/util/bip32.h index 7e58b79f38..347e83db9e 100644 --- a/src/util/bip32.h +++ b/src/util/bip32.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/golombrice.h b/src/util/golombrice.h new file mode 100644 index 0000000000..425e7f6681 --- /dev/null +++ b/src/util/golombrice.h @@ -0,0 +1,43 @@ +// Copyright (c) 2018-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_GOLOMBRICE_H +#define BITCOIN_UTIL_GOLOMBRICE_H + +#include <streams.h> + +#include <cstdint> + +template <typename OStream> +void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x) +{ + // Write quotient as unary-encoded: q 1's followed by one 0. + uint64_t q = x >> P; + while (q > 0) { + int nbits = q <= 64 ? static_cast<int>(q) : 64; + bitwriter.Write(~0ULL, nbits); + q -= nbits; + } + bitwriter.Write(0, 1); + + // Write the remainder in P bits. Since the remainder is just the bottom + // P bits of x, there is no need to mask first. + bitwriter.Write(x, P); +} + +template <typename IStream> +uint64_t GolombRiceDecode(BitStreamReader<IStream>& bitreader, uint8_t P) +{ + // Read unary-encoded quotient: q 1's followed by one 0. + uint64_t q = 0; + while (bitreader.Read(1) == 1) { + ++q; + } + + uint64_t r = bitreader.Read(P); + + return (q << P) + r; +} + +#endif // BITCOIN_UTIL_GOLOMBRICE_H diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index 544cfb58f9..1bc8d02eab 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/moneystr.h b/src/util/moneystr.h index d8b08adc24..9d2b6da0fc 100644 --- a/src/util/moneystr.h +++ b/src/util/moneystr.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 16917ccb47..3a903b6897 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 11317f0432..bd988f1410 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/string.h b/src/util/string.h index 694f0a1ca4..b8e2a06235 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/system.cpp b/src/util/system.cpp index b0a538b527..0790ea0d48 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -141,6 +141,12 @@ bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes) return free_bytes_available >= min_disk_space + additional_bytes; } +std::streampos GetFileSize(const char* path, std::streamsize max) { + std::ifstream file(path, std::ios::binary); + file.ignore(max); + return file.gcount(); +} + /** * Interpret a string argument as a boolean. * @@ -226,10 +232,11 @@ static bool CheckValid(const std::string& key, const util::SettingsValue& val, u return true; } -ArgsManager::ArgsManager() -{ - // nothing to do -} +// Define default constructor and destructor that are not inline, so code instantiating this class doesn't need to +// #include class definitions for all members. +// For example, m_settings has an internal dependency on univalue. +ArgsManager::ArgsManager() {} +ArgsManager::~ArgsManager() {} const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const { diff --git a/src/util/system.h b/src/util/system.h index 3138522b5c..a5eea5dfab 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -63,6 +63,14 @@ void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name bool DirIsWritable(const fs::path& directory); bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0); +/** Get the size of a file by scanning it. + * + * @param[in] path The file path + * @param[in] max Stop seeking beyond this limit + * @return The file size or max + */ +std::streampos GetFileSize(const char* path, std::streamsize max = std::numeric_limits<std::streamsize>::max()); + /** Release all directory locks. This is used for unit testing only, at runtime * the global destructor will take care of the locks. */ @@ -192,6 +200,7 @@ protected: public: ArgsManager(); + ~ArgsManager(); /** * Select the network in use diff --git a/src/util/time.cpp b/src/util/time.cpp index 0938ff36a6..e96972fe12 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/time.h b/src/util/time.h index 77de1e047d..b00c25f67c 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/url.h b/src/util/url.h index e9ea2ab765..5a7b11fa04 100644 --- a/src/util/url.h +++ b/src/util/url.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,8 @@ #include <string> -std::string urlDecode(const std::string &urlEncoded); +using UrlDecodeFn = std::string(const std::string& url_encoded); +UrlDecodeFn urlDecode; +extern UrlDecodeFn* const URL_DECODE; #endif // BITCOIN_UTIL_URL_H diff --git a/src/validation.cpp b/src/validation.cpp index 7ee94f8657..25975e3e31 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -20,6 +20,7 @@ #include <index/txindex.h> #include <logging.h> #include <logging/timer.h> +#include <optional.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -58,6 +59,25 @@ #define MICRO 0.000001 #define MILLI 0.001 +/** + * An extra transaction can be added to a package, as long as it only has one + * ancestor and is no larger than this. Not really any reason to make this + * configurable as it doesn't materially change DoS parameters. + */ +static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; +/** Maximum kilobytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; +/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ +static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ +static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** Time to wait (in seconds) between writing blocks/block index to disk. */ +static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; +/** Time to wait (in seconds) between flushing chainstate to disk. */ +static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; +/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ +static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; + bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { // First sort by most total work, ... if (pa->nChainWork > pb->nChainWork) return false; @@ -76,20 +96,19 @@ bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIn return false; } -namespace { -BlockManager g_blockman; -} // anon namespace - -std::unique_ptr<CChainState> g_chainstate; +ChainstateManager g_chainman; -CChainState& ChainstateActive() { - assert(g_chainstate); - return *g_chainstate; +CChainState& ChainstateActive() +{ + LOCK(::cs_main); + assert(g_chainman.m_active_chainstate); + return *g_chainman.m_active_chainstate; } -CChain& ChainActive() { - assert(g_chainstate); - return g_chainstate->m_chain; +CChain& ChainActive() +{ + LOCK(::cs_main); + return ::ChainstateActive().m_chain; } /** @@ -151,8 +170,8 @@ namespace { CBlockIndex* LookupBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); - BlockMap::const_iterator it = g_blockman.m_block_index.find(hash); - return it == g_blockman.m_block_index.end() ? nullptr : it->second; + BlockMap::const_iterator it = g_chainman.BlockIndex().find(hash); + return it == g_chainman.BlockIndex().end() ? nullptr : it->second; } CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) @@ -1016,7 +1035,7 @@ bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs // scripts (ie, other policy checks pass). We perform the inexpensive // checks first and avoid hashing and signature verification unless those // checks pass, to mitigate CPU exhaustion denial-of-service attacks. - PrecomputedTransactionData txdata(*ptx); + PrecomputedTransactionData txdata; if (!PolicyScriptChecks(args, workspace, txdata)) return false; @@ -1242,10 +1261,9 @@ void CoinsViews::InitCache() m_cacheview = MakeUnique<CCoinsViewCache>(&m_catcherview); } -// NOTE: for now m_blockman is set to a global, but this will be changed -// in a future commit. -CChainState::CChainState() : m_blockman(g_blockman) {} - +CChainState::CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash) + : m_blockman(blockman), + m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( size_t cache_size_bytes, @@ -1253,6 +1271,10 @@ void CChainState::InitCoinsDB( bool should_wipe, std::string leveldb_name) { + if (!m_from_snapshot_blockhash.IsNull()) { + leveldb_name += "_" + m_from_snapshot_blockhash.ToString(); + } + m_coins_views = MakeUnique<CoinsViews>( leveldb_name, cache_size_bytes, in_memory, should_wipe); } @@ -1294,7 +1316,8 @@ static CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; BlockMap& BlockIndex() { - return g_blockman.m_block_index; + LOCK(::cs_main); + return g_chainman.m_blockman.m_block_index; } static void AlertNotify(const std::string& strMessage) @@ -1512,6 +1535,10 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const C return true; } + if (!txdata.m_ready) { + txdata.Init(tx); + } + for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; const Coin& coin = inputs.AccessCoin(prevout); @@ -1668,10 +1695,11 @@ int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) return DISCONNECT_FAILED; // adding output for transaction without known metadata } } - // The potential_overwrite parameter to AddCoin is only allowed to be false if we know for - // sure that the coin did not already exist in the cache. As we have queried for that above - // using HaveCoin, we don't need to guess. When fClean is false, a coin already existed and - // it is an overwrite. + // If the coin already exists as an unspent coin in the cache, then the + // possible_overwrite parameter to AddCoin must be set to true. We have + // already checked whether an unspent coin exists above using HaveCoin, so + // we don't need to guess. When fClean is false, an unspent coin already + // existed and it is an overwrite. view.AddCoin(out, std::move(undo), !fClean); return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; @@ -2075,15 +2103,19 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockUndo blockundo; + // Precomputed transaction data pointers must not be invalidated + // until after `control` has run the script checks (potentially + // in multiple threads). Preallocate the vector size so a new allocation + // doesn't invalidate pointers into the vector, and keep txsdata in scope + // for as long as `control`. CCheckQueueControl<CScriptCheck> control(fScriptChecks && g_parallel_script_checks ? &scriptcheckqueue : nullptr); + std::vector<PrecomputedTransactionData> txsdata(block.vtx.size()); std::vector<int> prevheights; CAmount nFees = 0; int nInputs = 0; int64_t nSigOpsCost = 0; blockundo.vtxundo.reserve(block.vtx.size() - 1); - std::vector<PrecomputedTransactionData> txdata; - txdata.reserve(block.vtx.size()); // Required so that pointers to individual PrecomputedTransactionData don't get invalidated for (unsigned int i = 0; i < block.vtx.size(); i++) { const CTransaction &tx = *(block.vtx[i]); @@ -2130,13 +2162,12 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops"); } - txdata.emplace_back(tx); if (!tx.IsCoinBase()) { std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ TxValidationState tx_state; - if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { + if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txsdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { // Any transaction validation failure in ConnectBlock is a block consensus failure state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), tx_state.GetDebugMessage()); @@ -3443,7 +3474,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio if (fCheckpointsEnabled) { // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our - // g_blockman.m_block_index. + // BlockIndex(). CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); @@ -3651,7 +3682,8 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValid LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast - bool accepted = g_blockman.AcceptBlockHeader(header, state, chainparams, &pindex); + bool accepted = g_chainman.m_blockman.AcceptBlockHeader( + header, state, chainparams, &pindex); ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); if (!accepted) { @@ -3853,7 +3885,7 @@ void PruneOneBlockFile(const int fileNumber) { LOCK(cs_LastBlockFile); - for (const auto& entry : g_blockman.m_block_index) { + for (const auto& entry : g_chainman.BlockIndex()) { CBlockIndex* pindex = entry.second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; @@ -3867,12 +3899,12 @@ void PruneOneBlockFile(const int fileNumber) // to be downloaded again in order to consider its chain, at which // point it would be considered as a candidate for // m_blocks_unlinked or setBlockIndexCandidates. - auto range = g_blockman.m_blocks_unlinked.equal_range(pindex->pprev); + auto range = g_chainman.m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); while (range.first != range.second) { std::multimap<CBlockIndex *, CBlockIndex *>::iterator _it = range.first; range.first++; if (_it->second == pindex) { - g_blockman.m_blocks_unlinked.erase(_it); + g_chainman.m_blockman.m_blocks_unlinked.erase(_it); } } } @@ -4109,9 +4141,11 @@ void BlockManager::Unload() { bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - if (!g_blockman.LoadBlockIndex( - chainparams.GetConsensus(), *pblocktree, ::ChainstateActive().setBlockIndexCandidates)) + if (!g_chainman.m_blockman.LoadBlockIndex( + chainparams.GetConsensus(), *pblocktree, + ::ChainstateActive().setBlockIndexCandidates)) { return false; + } // Load block file info pblocktree->ReadLastBlockFile(nLastBlockFile); @@ -4133,7 +4167,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - for (const std::pair<const uint256, CBlockIndex*>& item : g_blockman.m_block_index) + for (const std::pair<const uint256, CBlockIndex*>& item : g_chainman.BlockIndex()) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { @@ -4510,26 +4544,15 @@ bool CChainState::RewindBlockIndex(const CChainParams& params) PruneBlockIndexCandidates(); CheckBlockIndex(params.GetConsensus()); - } - } - - return true; -} -bool RewindBlockIndex(const CChainParams& params) { - if (!::ChainstateActive().RewindBlockIndex(params)) { - return false; - } - - LOCK(cs_main); - if (::ChainActive().Tip() != nullptr) { - // FlushStateToDisk can possibly read ::ChainActive(). Be conservative - // and skip it here, we're about to -reindex-chainstate anyway, so - // it'll get called a bunch real soon. - BlockValidationState state; - if (!::ChainstateActive().FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { - LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); - return false; + // FlushStateToDisk can possibly read ::ChainActive(). Be conservative + // and skip it here, we're about to -reindex-chainstate anyway, so + // it'll get called a bunch real soon. + BlockValidationState state; + if (!FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { + LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); + return false; + } } } @@ -4547,8 +4570,7 @@ void CChainState::UnloadBlockIndex() { void UnloadBlockIndex() { LOCK(cs_main); - ::ChainActive().SetTip(nullptr); - g_blockman.Unload(); + g_chainman.Unload(); pindexBestInvalid = nullptr; pindexBestHeader = nullptr; mempool.clear(); @@ -4561,8 +4583,6 @@ void UnloadBlockIndex() warningcache[b].clear(); } fHavePruned = false; - - ::ChainstateActive().UnloadBlockIndex(); } bool LoadBlockIndex(const CChainParams& chainparams) @@ -4572,7 +4592,7 @@ bool LoadBlockIndex(const CChainParams& chainparams) if (!fReindex) { bool ret = LoadBlockIndexDB(chainparams); if (!ret) return false; - needs_init = g_blockman.m_block_index.empty(); + needs_init = g_chainman.m_blockman.m_block_index.empty(); } if (needs_init) { @@ -4693,7 +4713,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // Activate the genesis block so normal node progress can continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { + if (!ActivateBestChain(state, chainparams, nullptr)) { break; } } @@ -4923,6 +4943,14 @@ void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) assert(nNodes == forward.size()); } +std::string CChainState::ToString() +{ + CBlockIndex* tip = m_chain.Tip(); + return strprintf("Chainstate [%s] @ height %d (%s)", + m_from_snapshot_blockhash.IsNull() ? "ibd" : "snapshot", + tip ? tip->nHeight : -1, tip ? tip->GetBlockHash().ToString() : "null"); +} + std::string CBlockFileInfo::ToString() const { return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); @@ -5110,10 +5138,99 @@ public: CMainCleanup() {} ~CMainCleanup() { // block headers - BlockMap::iterator it1 = g_blockman.m_block_index.begin(); - for (; it1 != g_blockman.m_block_index.end(); it1++) + BlockMap::iterator it1 = g_chainman.BlockIndex().begin(); + for (; it1 != g_chainman.BlockIndex().end(); it1++) delete (*it1).second; - g_blockman.m_block_index.clear(); + g_chainman.BlockIndex().clear(); } }; static CMainCleanup instance_of_cmaincleanup; + +Optional<uint256> ChainstateManager::SnapshotBlockhash() const { + if (m_active_chainstate != nullptr) { + // If a snapshot chainstate exists, it will always be our active. + return m_active_chainstate->m_from_snapshot_blockhash; + } + return {}; +} + +std::vector<CChainState*> ChainstateManager::GetAll() +{ + std::vector<CChainState*> out; + + if (!IsSnapshotValidated() && m_ibd_chainstate) { + out.push_back(m_ibd_chainstate.get()); + } + + if (m_snapshot_chainstate) { + out.push_back(m_snapshot_chainstate.get()); + } + + return out; +} + +CChainState& ChainstateManager::InitializeChainstate(const uint256& snapshot_blockhash) +{ + bool is_snapshot = !snapshot_blockhash.IsNull(); + std::unique_ptr<CChainState>& to_modify = + is_snapshot ? m_snapshot_chainstate : m_ibd_chainstate; + + if (to_modify) { + throw std::logic_error("should not be overwriting a chainstate"); + } + + to_modify.reset(new CChainState(m_blockman, snapshot_blockhash)); + + // Snapshot chainstates and initial IBD chaintates always become active. + if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { + LogPrintf("Switching active chainstate to %s\n", to_modify->ToString()); + m_active_chainstate = to_modify.get(); + } else { + throw std::logic_error("unexpected chainstate activation"); + } + + return *to_modify; +} + +CChain& ChainstateManager::ActiveChain() const +{ + assert(m_active_chainstate); + return m_active_chainstate->m_chain; +} + +bool ChainstateManager::IsSnapshotActive() const +{ + return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); +} + +CChainState& ChainstateManager::ValidatedChainstate() const +{ + if (m_snapshot_chainstate && IsSnapshotValidated()) { + return *m_snapshot_chainstate.get(); + } + assert(m_ibd_chainstate); + return *m_ibd_chainstate.get(); +} + +bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const +{ + return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); +} + +void ChainstateManager::Unload() +{ + for (CChainState* chainstate : this->GetAll()) { + chainstate->m_chain.SetTip(nullptr); + chainstate->UnloadBlockIndex(); + } + + m_blockman.Unload(); +} + +void ChainstateManager::Reset() +{ + m_ibd_chainstate.reset(); + m_snapshot_chainstate.reset(); + m_active_chainstate = nullptr; + m_snapshot_validated = false; +} diff --git a/src/validation.h b/src/validation.h index a5335edc43..91b1ba6497 100644 --- a/src/validation.h +++ b/src/validation.h @@ -14,6 +14,7 @@ #include <coins.h> #include <crypto/common.h> // for ReadLE64 #include <fs.h> +#include <optional.h> #include <policy/feerate.h> #include <protocol.h> // For CMessageHeader::MessageStartChars #include <script/script_error.h> @@ -59,57 +60,15 @@ static const unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT = 101; static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25; /** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */ static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; -/** - * An extra transaction can be added to a package, as long as it only has one - * ancestor and is no larger than this. Not really any reason to make this - * configurable as it doesn't materially change DoS parameters. - */ -static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; -/** Maximum kilobytes for transactions to store for processing during reorg */ -static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB -/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ -static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB -/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ -static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB - /** Maximum number of dedicated script-checking threads allowed */ static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; -/** Number of blocks that can be requested at any given time from a single peer. */ -static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; -/** Timeout in seconds during which a peer must stall block download progress before being disconnected. */ -static const unsigned int BLOCK_STALLING_TIMEOUT = 2; -/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends - * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ -static const unsigned int MAX_HEADERS_RESULTS = 2000; -/** Maximum depth of blocks we're willing to serve as compact blocks to peers - * when requested. For older blocks, a regular BLOCK response will be sent. */ -static const int MAX_CMPCTBLOCK_DEPTH = 5; -/** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ -static const int MAX_BLOCKTXN_DEPTH = 10; -/** Size of the "block download window": how far ahead of our current height do we fetch? - * Larger windows tolerate larger download speed differences between peer, but increase the potential - * degree of disordering of blocks on disk (which make reindexing and pruning harder). We'll probably - * want to make this a per-peer adaptive value at some point. */ -static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; -/** Time to wait (in seconds) between writing blocks/block index to disk. */ -static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; -/** Time to wait (in seconds) between flushing chainstate to disk. */ -static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; -/** Block download timeout base, expressed in millionths of the block interval (i.e. 10 min) */ -static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; -/** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ -static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; - static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; -/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ -static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; - static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; @@ -118,15 +77,21 @@ static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for using fee filter */ static const bool DEFAULT_FEEFILTER = true; - -/** Maximum number of headers to announce when relaying blocks with headers message.*/ -static const unsigned int MAX_BLOCKS_TO_ANNOUNCE = 8; - -/** Maximum number of unconnecting headers announcements before DoS score */ -static const int MAX_UNCONNECTING_HEADERS = 10; - /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; +/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ +static const unsigned int MIN_BLOCKS_TO_KEEP = 288; +static const signed int DEFAULT_CHECKBLOCKS = 6; +static const unsigned int DEFAULT_CHECKLEVEL = 3; +// Require that user allocate at least 550 MiB for block & undo files (blk???.dat and rev???.dat) +// At 1MB per block, 288 blocks = 288MB. +// Add 15% for Undo data = 331MB +// Add 20% for Orphan block rate = 397MB +// We want the low water mark after pruning to be at least 397 MB and since we prune in +// full block file chunks, we need the high water mark which triggers the prune to be +// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB +// Setting the target to >= 550 MiB will make it likely we can respect the target. +static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; struct BlockHasher { @@ -174,23 +139,6 @@ extern bool fHavePruned; extern bool fPruneMode; /** Number of MiB of block files that we're trying to stay below. */ extern uint64_t nPruneTarget; -/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ -static const unsigned int MIN_BLOCKS_TO_KEEP = 288; -/** Minimum blocks required to signal NODE_NETWORK_LIMITED */ -static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; - -static const signed int DEFAULT_CHECKBLOCKS = 6; -static const unsigned int DEFAULT_CHECKLEVEL = 3; - -// Require that user allocate at least 550 MiB for block & undo files (blk???.dat and rev???.dat) -// At 1MB per block, 288 blocks = 288MB. -// Add 15% for Undo data = 331MB -// Add 20% for Orphan block rate = 397MB -// We want the low water mark after pruning to be at least 397 MB and since we prune in -// full block file chunks, we need the high water mark which triggers the prune to be -// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB -// Setting the target to >= 550 MiB will make it likely we can respect the target. -static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; /** * Process an incoming block. This only returns after the best known valid @@ -379,9 +327,6 @@ bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainpar * Note that transaction witness validation rules are always enforced when P2SH is enforced. */ bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& params); -/** When there are blocks in the active chain with missing data, rewind the chainstate and remove them from the block index */ -bool RewindBlockIndex(const CChainParams& params) LOCKS_EXCLUDED(cs_main); - /** Compute at which vout of the block's coinbase transaction the witness commitment occurs, or -1 if not found */ int GetWitnessCommitmentIndex(const CBlock& block); @@ -539,6 +484,9 @@ enum class CoinsCacheSizeState OK = 0 }; +// Defined below, but needed for `friend` usage in CChainState. +class ChainstateManager; + /** * CChainState stores and provides an API to update our local knowledge of the * current best chain. @@ -591,8 +539,7 @@ private: std::unique_ptr<CoinsViews> m_coins_views; public: - CChainState(BlockManager& blockman) : m_blockman(blockman) {} - CChainState(); + explicit CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash = uint256()); /** * Initialize the CoinsViews UTXO set database management data structures. The in-memory @@ -621,6 +568,13 @@ public: CChain m_chain; /** + * The blockhash which is the base of the snapshot this chainstate was created from. + * + * IsNull() if this chainstate was not created from a snapshot. + */ + const uint256 m_from_snapshot_blockhash{}; + + /** * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be * missing the data for the block. @@ -741,6 +695,8 @@ public: size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + std::string ToString() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + private: bool ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); bool ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); @@ -753,6 +709,8 @@ private: //! Mark a block as not having block data void EraseBlockData(CBlockIndex* index) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + friend ChainstateManager; }; /** Mark a block as precious and reorganize. @@ -768,6 +726,150 @@ bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparam /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +/** + * Provides an interface for creating and interacting with one or two + * chainstates: an IBD chainstate generated by downloading blocks, and + * an optional snapshot chainstate loaded from a UTXO snapshot. Managed + * chainstates can be maintained at different heights simultaneously. + * + * This class provides abstractions that allow the retrieval of the current + * most-work chainstate ("Active") as well as chainstates which may be in + * background use to validate UTXO snapshots. + * + * Definitions: + * + * *IBD chainstate*: a chainstate whose current state has been "fully" + * validated by the initial block download process. + * + * *Snapshot chainstate*: a chainstate populated by loading in an + * assumeutxo UTXO snapshot. + * + * *Active chainstate*: the chainstate containing the current most-work + * chain. Consulted by most parts of the system (net_processing, + * wallet) as a reflection of the current chain and UTXO set. + * This may either be an IBD chainstate or a snapshot chainstate. + * + * *Background IBD chainstate*: an IBD chainstate for which the + * IBD process is happening in the background while use of the + * active (snapshot) chainstate allows the rest of the system to function. + * + * *Validated chainstate*: the most-work chainstate which has been validated + * locally via initial block download. This will be the snapshot chainstate + * if a snapshot was loaded and all blocks up to the snapshot starting point + * have been downloaded and validated (via background validation), otherwise + * it will be the IBD chainstate. + */ +class ChainstateManager +{ +private: + //! The chainstate used under normal operation (i.e. "regular" IBD) or, if + //! a snapshot is in use, for background validation. + //! + //! Its contents (including on-disk data) will be deleted *upon shutdown* + //! after background validation of the snapshot has completed. We do not + //! free the chainstate contents immediately after it finishes validation + //! to cautiously avoid a case where some other part of the system is still + //! using this pointer (e.g. net_processing). + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + std::unique_ptr<CChainState> m_ibd_chainstate; + + //! A chainstate initialized on the basis of a UTXO snapshot. If this is + //! non-null, it is always our active chainstate. + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + std::unique_ptr<CChainState> m_snapshot_chainstate; + + //! Points to either the ibd or snapshot chainstate; indicates our + //! most-work chain. + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + CChainState* m_active_chainstate{nullptr}; + + //! If true, the assumed-valid chainstate has been fully validated + //! by the background validation chainstate. + bool m_snapshot_validated{false}; + + // For access to m_active_chainstate. + friend CChainState& ChainstateActive(); + friend CChain& ChainActive(); + +public: + //! A single BlockManager instance is shared across each constructed + //! chainstate to avoid duplicating block metadata. + BlockManager m_blockman GUARDED_BY(::cs_main); + + //! Instantiate a new chainstate and assign it based upon whether it is + //! from a snapshot. + //! + //! @param[in] snapshot_blockhash If given, signify that this chainstate + //! is based on a snapshot. + CChainState& InitializeChainstate(const uint256& snapshot_blockhash = uint256()) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Get all chainstates currently being used. + std::vector<CChainState*> GetAll(); + + //! The most-work chain. + CChain& ActiveChain() const; + int ActiveHeight() const { return ActiveChain().Height(); } + CBlockIndex* ActiveTip() const { return ActiveChain().Tip(); } + + BlockMap& BlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + { + return m_blockman.m_block_index; + } + + bool IsSnapshotActive() const; + + Optional<uint256> SnapshotBlockhash() const; + + //! Is there a snapshot in use and has it been fully validated? + bool IsSnapshotValidated() const { return m_snapshot_validated; } + + //! @returns true if this chainstate is being used to validate an active + //! snapshot in the background. + bool IsBackgroundIBD(CChainState* chainstate) const; + + //! Return the most-work chainstate that has been fully validated. + //! + //! During background validation of a snapshot, this is the IBD chain. After + //! background validation has completed, this is the snapshot chain. + CChainState& ValidatedChainstate() const; + + CChain& ValidatedChain() const { return ValidatedChainstate().m_chain; } + CBlockIndex* ValidatedTip() const { return ValidatedChain().Tip(); } + + //! Unload block index and chain data before shutdown. + void Unload() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Clear (deconstruct) chainstate data. + void Reset(); +}; + +extern ChainstateManager g_chainman GUARDED_BY(::cs_main); + /** @returns the most-work valid chainstate. */ CChainState& ChainstateActive(); @@ -777,11 +879,6 @@ CChain& ChainActive(); /** @returns the global block index map. */ BlockMap& BlockIndex(); -// Most often ::ChainstateActive() should be used instead of this, but some code -// may not be able to assume that this has been initialized yet and so must use it -// directly, e.g. init.cpp. -extern std::unique_ptr<CChainState> g_chainstate; - /** Global variable that points to the active block tree (protected by cs_main) */ extern std::unique_ptr<CBlockTreeDB> pblocktree; diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index c06647cb0d..11000774c0 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -67,8 +67,8 @@ public: void Clear() { LOCK(m_mutex); - for (auto it = m_list.begin(); it != m_list.end();) { - it = --it->count ? std::next(it) : m_list.erase(it); + for (const auto& entry : m_map) { + if (!--entry.second->count) m_list.erase(entry.second); } m_map.clear(); } diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 5bbb2c0ad0..079a5d3d53 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -106,6 +106,9 @@ bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& target_v best_selection = curr_selection; best_selection.resize(utxo_pool.size()); best_waste = curr_waste; + if (best_waste == 0) { + break; + } } curr_waste -= (curr_value - actual_target); // Remove the excess value as we will be selecting different coins now backtrack = true; diff --git a/src/wallet/db.h b/src/wallet/db.h index bebaa55d05..1c5f42abca 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 1623ab9074..dafa022ded 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index 859f754761..fc038ae731 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 50f064b305..8688f3df5e 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -57,7 +57,6 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-salvagewallet", "Attempt to recover private keys from a corrupt wallet on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-spendzeroconfchange", strprintf("Spend unconfirmed change when sending transactions (default: %u)", DEFAULT_SPEND_ZEROCONF_CHANGE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-txconfirmtarget=<n>", strprintf("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)", DEFAULT_TX_CONFIRM_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-upgradewallet", "Upgrade wallet to latest format on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-wallet=<path>", "Specify wallet database path. Can be specified multiple times to load multiple wallets. Path is interpreted relative to <walletdir> if it is not absolute, and will be created if it does not exist (as a directory containing a wallet.dat file and log files). For backwards compatibility this will also accept names of existing data files in <walletdir>.)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); gArgs.AddArg("-walletbroadcast", strprintf("Make the wallet broadcast transactions (default: %u)", DEFAULT_WALLETBROADCAST), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-walletdir=<dir>", "Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); @@ -116,12 +115,6 @@ bool WalletInit::ParameterInteraction() const } } - if (is_multiwallet) { - if (gArgs.GetBoolArg("-upgradewallet", false)) { - return InitError(strprintf("%s is only allowed with a single wallet file", "-upgradewallet")); - } - } - if (gArgs.GetBoolArg("-sysperms", false)) return InitError("-sysperms is not allowed in combination with enabled wallet functionality"); diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index d6e44c7be5..0afd2dfcf0 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index ea54027c48..9417e2bd58 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,6 +28,8 @@ +using interfaces::FoundBlock; + std::string static EncodeDumpString(const std::string &str) { std::stringstream ret; for (const unsigned char c : str) { @@ -128,7 +130,7 @@ UniValue importprivkey(const JSONRPCRequest& request) EnsureLegacyScriptPubKeyMan(*wallet, true); - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); bool fRescan = true; { auto locked_chain = pwallet->chain().lock(); @@ -272,7 +274,7 @@ UniValue importaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -360,8 +362,9 @@ UniValue importprunedfunds(const JSONRPCRequest& request) } auto locked_chain = pwallet->chain().lock(); - Optional<int> height = locked_chain->getBlockHeight(merkleBlock.header.GetHash()); - if (height == nullopt) { + LOCK(pwallet->cs_wallet); + int height; + if (!pwallet->chain().findAncestorByHash(pwallet->GetLastBlockHash(), merkleBlock.header.GetHash(), FoundBlock().height(height))) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); } @@ -372,11 +375,9 @@ UniValue importprunedfunds(const JSONRPCRequest& request) unsigned int txnIndex = vIndex[it - vMatch.begin()]; - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *height, merkleBlock.header.GetHash(), txnIndex); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, height, merkleBlock.header.GetHash(), txnIndex); wtx.m_confirm = confirm; - LOCK(pwallet->cs_wallet); - if (pwallet->IsMine(*wtx.tx)) { pwallet->AddToWallet(wtx, false); return NullUniValue; @@ -473,7 +474,7 @@ UniValue importpubkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -548,7 +549,7 @@ UniValue importwallet(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -566,8 +567,7 @@ UniValue importwallet(const JSONRPCRequest& request) if (!file.is_open()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); } - Optional<int> tip_height = locked_chain->getHeight(); - nTimeBegin = tip_height ? locked_chain->getBlockTime(*tip_height) : 0; + CHECK_NONFATAL(pwallet->chain().findBlock(pwallet->GetLastBlockHash(), FoundBlock().time(nTimeBegin))); int64_t nFilesize = std::max((int64_t)1, (int64_t)file.tellg()); file.seekg(0, file.beg); @@ -724,9 +724,8 @@ UniValue dumpprivkey(const JSONRPCRequest& request) UniValue dumpwallet(const JSONRPCRequest& request) { - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - const CWallet* const pwallet = wallet.get(); - if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet.get(), request.fHelp)) { return NullUniValue; } @@ -736,7 +735,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) "Note that if your wallet contains keys which are not derived from your HD seed (e.g. imported keys), these are not covered by\n" "only backing up the seed itself, and must be backed up too (e.g. ensure you back up the whole dumpfile).\n", { - {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The filename with path (either absolute or relative to bitcoind)"}, + {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The filename with path (absolute path recommended)"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -750,12 +749,17 @@ UniValue dumpwallet(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); + CWallet& wallet = *pwallet; + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(wallet); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + wallet.BlockUntilSyncedToCurrentChain(); auto locked_chain = pwallet->chain().lock(); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); - EnsureWalletIsUnlocked(pwallet); + EnsureWalletIsUnlocked(&wallet); fs::path filepath = request.params[0].get_str(); filepath = fs::absolute(filepath); @@ -791,9 +795,10 @@ UniValue dumpwallet(const JSONRPCRequest& request) // produce output file << strprintf("# Wallet dump created by Bitcoin %s\n", CLIENT_BUILD); file << strprintf("# * Created on %s\n", FormatISO8601DateTime(GetTime())); - const Optional<int> tip_height = locked_chain->getHeight(); - file << strprintf("# * Best block at time of backup was %i (%s),\n", tip_height.get_value_or(-1), tip_height ? locked_chain->getBlockHash(*tip_height).ToString() : "(missing block hash)"); - file << strprintf("# mined on %s\n", tip_height ? FormatISO8601DateTime(locked_chain->getBlockTime(*tip_height)) : "(missing block time)"); + file << strprintf("# * Best block at time of backup was %i (%s),\n", wallet.GetLastBlockHeight(), wallet.GetLastBlockHash().ToString()); + int64_t block_time = 0; + CHECK_NONFATAL(wallet.chain().findBlock(wallet.GetLastBlockHash(), FoundBlock().time(block_time))); + file << strprintf("# mined on %s\n", FormatISO8601DateTime(block_time)); file << "\n"; // add the base58check encoded extended master if the wallet uses HD @@ -816,8 +821,8 @@ UniValue dumpwallet(const JSONRPCRequest& request) CKey key; if (spk_man.GetKey(keyid, key)) { file << strprintf("%s %s ", EncodeSecret(key), strTime); - if (GetWalletAddressesForKey(&spk_man, pwallet, keyid, strAddr, strLabel)) { - file << strprintf("label=%s", strLabel); + if (GetWalletAddressesForKey(&spk_man, &wallet, keyid, strAddr, strLabel)) { + file << strprintf("label=%s", strLabel); } else if (keyid == seed_id) { file << "hdseed=1"; } else if (mapKeyPool.count(keyid)) { @@ -1364,7 +1369,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -1379,20 +1384,13 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) EnsureWalletIsUnlocked(pwallet); // Verify all timestamps are present before importing any keys. - const Optional<int> tip_height = locked_chain->getHeight(); - now = tip_height ? locked_chain->getBlockMedianTimePast(*tip_height) : 0; + CHECK_NONFATAL(pwallet->chain().findBlock(pwallet->GetLastBlockHash(), FoundBlock().time(nLowestTimestamp).mtpTime(now))); for (const UniValue& data : requests.getValues()) { GetImportTimestamp(data, now); } const int64_t minimumTimestamp = 1; - if (fRescan && tip_height) { - nLowestTimestamp = locked_chain->getBlockTime(*tip_height); - } else { - fRescan = false; - } - for (const UniValue& data : requests.getValues()) { const int64_t timestamp = std::max(GetImportTimestamp(data, now), minimumTimestamp); const UniValue result = ProcessImport(pwallet, data, timestamp); @@ -1460,3 +1458,297 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) return response; } + +static UniValue ProcessDescriptorImport(CWallet * const pwallet, const UniValue& data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + UniValue warnings(UniValue::VARR); + UniValue result(UniValue::VOBJ); + + try { + if (!data.exists("desc")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Descriptor not found."); + } + + const std::string& descriptor = data["desc"].get_str(); + const bool active = data.exists("active") ? data["active"].get_bool() : false; + const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; + const std::string& label = data.exists("label") ? data["label"].get_str() : ""; + + // Parse descriptor string + FlatSigningProvider keys; + std::string error; + auto parsed_desc = Parse(descriptor, keys, error, /* require_checksum = */ true); + if (!parsed_desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); + } + + // Range check + int64_t range_start = 0, range_end = 1, next_index = 0; + if (!parsed_desc->IsRange() && data.exists("range")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for an un-ranged descriptor"); + } else if (parsed_desc->IsRange()) { + if (data.exists("range")) { + auto range = ParseDescriptorRange(data["range"]); + range_start = range.first; + range_end = range.second + 1; // Specified range end is inclusive, but we need range end as exclusive + } else { + warnings.push_back("Range not given, using default keypool range"); + range_start = 0; + range_end = gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE); + } + next_index = range_start; + + if (data.exists("next_index")) { + next_index = data["next_index"].get_int64(); + // bound checks + if (next_index < range_start || next_index >= range_end) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "next_index is out of range"); + } + } + } + + // Active descriptors must be ranged + if (active && !parsed_desc->IsRange()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Active descriptors must be ranged"); + } + + // Ranged descriptors should not have a label + if (data.exists("range") && data.exists("label")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptors should not have a label"); + } + + // Internal addresses should not have a label either + if (internal && data.exists("label")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal addresses should not have a label"); + } + + // Combo descriptor check + if (active && !parsed_desc->IsSingleType()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Combo descriptors cannot be set to active"); + } + + // If the wallet disabled private keys, abort if private keys exist + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !keys.keys.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); + } + + // Need to ExpandPrivate to check if private keys are available for all pubkeys + FlatSigningProvider expand_keys; + std::vector<CScript> scripts; + parsed_desc->Expand(0, keys, scripts, expand_keys); + parsed_desc->ExpandPrivate(0, keys, expand_keys); + + // Check if all private keys are provided + bool have_all_privkeys = !expand_keys.keys.empty(); + for (const auto& entry : expand_keys.origins) { + const CKeyID& key_id = entry.first; + CKey key; + if (!expand_keys.GetKey(key_id, key)) { + have_all_privkeys = false; + break; + } + } + + // If private keys are enabled, check some things. + if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (keys.keys.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import descriptor without private keys to a wallet with private keys enabled"); + } + if (!have_all_privkeys) { + warnings.push_back("Not all private keys provided. Some wallet functionality may return unexpected errors"); + } + } + + WalletDescriptor w_desc(std::move(parsed_desc), timestamp, range_start, range_end, next_index); + + // Check if the wallet already contains the descriptor + auto existing_spk_manager = pwallet->GetDescriptorScriptPubKeyMan(w_desc); + if (existing_spk_manager) { + LOCK(existing_spk_manager->cs_desc_man); + if (range_start > existing_spk_manager->GetWalletDescriptor().range_start) { + throw JSONRPCError(RPC_INVALID_PARAMS, strprintf("range_start can only decrease; current range = [%d,%d]", existing_spk_manager->GetWalletDescriptor().range_start, existing_spk_manager->GetWalletDescriptor().range_end)); + } + } + + // Add descriptor to the wallet + auto spk_manager = pwallet->AddWalletDescriptor(w_desc, keys, label); + if (spk_manager == nullptr) { + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Could not add descriptor '%s'", descriptor)); + } + + // Set descriptor as active if necessary + if (active) { + if (!w_desc.descriptor->GetOutputType()) { + warnings.push_back("Unknown output type, cannot set descriptor to active."); + } else { + pwallet->SetActiveScriptPubKeyMan(spk_manager->GetID(), *w_desc.descriptor->GetOutputType(), internal); + } + } + + result.pushKV("success", UniValue(true)); + } catch (const UniValue& e) { + result.pushKV("success", UniValue(false)); + result.pushKV("error", e); + } catch (...) { + result.pushKV("success", UniValue(false)); + + result.pushKV("error", JSONRPCError(RPC_MISC_ERROR, "Missing required fields")); + } + if (warnings.size()) result.pushKV("warnings", warnings); + return result; +} + +UniValue importdescriptors(const JSONRPCRequest& main_request) { + // Acquire the wallet + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(main_request); + CWallet* const pwallet = wallet.get(); + if (!EnsureWalletIsAvailable(pwallet, main_request.fHelp)) { + return NullUniValue; + } + + RPCHelpMan{"importdescriptors", + "\nImport descriptors. This will trigger a rescan of the blockchain based on the earliest timestamp of all descriptors being imported. Requires a new wallet backup.\n" + "\nNote: This call can take over an hour to complete if using an early timestamp; during that time, other rpc calls\n" + "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n", + { + {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", + { + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + { + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "Descriptor to import."}, + {"active", RPCArg::Type::BOOL, /* default */ "false", "Set this descriptor to be the active descriptor for the corresponding output type/externality"}, + {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED, "If a ranged descriptor is used, this specifies the end or the range (in the form [begin,end]) to import"}, + {"next_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "If a ranged descriptor is set to active, this specifies the next index to generate addresses from"}, + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, "Time from which to start rescanning the blockchain for this descriptor, in " + UNIX_EPOCH_TIME + "\n" + " Use the string \"now\" to substitute the current synced blockchain time.\n" + " \"now\" can be specified to bypass scanning, for outputs which are known to never have been used, and\n" + " 0 can be specified to scan the entire blockchain. Blocks up to 2 hours before the earliest timestamp\n" + " of all descriptors being imported will be scanned.", + /* oneline_description */ "", {"timestamp | \"now\"", "integer / string"} + }, + {"internal", RPCArg::Type::BOOL, /* default */ "false", "Whether matching outputs should be treated as not incoming payments (e.g. change)"}, + {"label", RPCArg::Type::STR, /* default */ "''", "Label to assign to the address, only allowed with internal=false"}, + }, + }, + }, + "\"requests\""}, + }, + RPCResult{ + RPCResult::Type::ARR, "", "Response is an array with the same size as the input that has the execution result", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "success", ""}, + {RPCResult::Type::ARR, "warnings", /* optional */ true, "", + { + {RPCResult::Type::STR, "", ""}, + }}, + {RPCResult::Type::OBJ, "error", /* optional */ true, "", + { + {RPCResult::Type::ELISION, "", "JSONRPC error"}, + }}, + }}, + } + }, + RPCExamples{ + HelpExampleCli("importdescriptors", "'[{ \"desc\": \"<my descriptor>\", \"timestamp\":1455191478, \"internal\": true }, " + "{ \"desc\": \"<my desccriptor 2>\", \"label\": \"example 2\", \"timestamp\": 1455191480 }]'") + + HelpExampleCli("importdescriptors", "'[{ \"desc\": \"<my descriptor>\", \"timestamp\":1455191478, \"active\": true, \"range\": [0,100], \"label\": \"<my bech32 wallet>\" }]'") + }, + }.Check(main_request); + + // Make sure wallet is a descriptor wallet + if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "importdescriptors is not available for non-descriptor wallets"); + } + + RPCTypeCheck(main_request.params, {UniValue::VARR, UniValue::VOBJ}); + + WalletRescanReserver reserver(*pwallet); + if (!reserver.reserve()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); + } + + const UniValue& requests = main_request.params[0]; + const int64_t minimum_timestamp = 1; + int64_t now = 0; + int64_t lowest_timestamp = 0; + bool rescan = false; + UniValue response(UniValue::VARR); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + EnsureWalletIsUnlocked(pwallet); + + CHECK_NONFATAL(pwallet->chain().findBlock(pwallet->GetLastBlockHash(), FoundBlock().time(lowest_timestamp).mtpTime(now))); + + // Get all timestamps and extract the lowest timestamp + for (const UniValue& request : requests.getValues()) { + // This throws an error if "timestamp" doesn't exist + const int64_t timestamp = std::max(GetImportTimestamp(request, now), minimum_timestamp); + const UniValue result = ProcessDescriptorImport(pwallet, request, timestamp); + response.push_back(result); + + if (lowest_timestamp > timestamp ) { + lowest_timestamp = timestamp; + } + + // If we know the chain tip, and at least one request was successful then allow rescan + if (!rescan && result["success"].get_bool()) { + rescan = true; + } + } + pwallet->ConnectScriptPubKeyManNotifiers(); + } + + // Rescan the blockchain using the lowest timestamp + if (rescan) { + int64_t scanned_time = pwallet->RescanFromTime(lowest_timestamp, reserver, true /* update */); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(); + } + + if (pwallet->IsAbortingRescan()) { + throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); + } + + if (scanned_time > lowest_timestamp) { + std::vector<UniValue> results = response.getValues(); + response.clear(); + response.setArray(); + + // Compose the response + for (unsigned int i = 0; i < requests.size(); ++i) { + const UniValue& request = requests.getValues().at(i); + + // If the descriptor timestamp is within the successfully scanned + // range, or if the import result already has an error set, let + // the result stand unmodified. Otherwise replace the result + // with an error message. + if (scanned_time <= GetImportTimestamp(request, now) || results.at(i).exists("error")) { + response.push_back(results.at(i)); + } else { + UniValue result = UniValue(UniValue::VOBJ); + result.pushKV("success", UniValue(false)); + result.pushKV( + "error", + JSONRPCError( + RPC_MISC_ERROR, + strprintf("Rescan failed for descriptor with timestamp %d. There was an error reading a " + "block from time %d, which is after or within %d seconds of key creation, and " + "could contain transactions pertaining to the desc. As a result, transactions " + "and coins using this desc may not appear in the wallet. This error could be " + "caused by pruning or data corruption (see bitcoind log for details) and could " + "be dealt with by downloading and rescanning the relevant blocks (see -reindex " + "and -rescan options).", + GetImportTimestamp(request, now), scanned_time - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW))); + response.push_back(std::move(result)); + } + } + } + } + + return response; +} diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 61ad2f1198..49c583c422 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -37,6 +37,8 @@ #include <univalue.h> +using interfaces::FoundBlock; + static const std::string WALLET_ENDPOINT_BASE = "/wallet/"; static const std::string HELP_REQUIRING_PASSPHRASE{"\nRequires wallet passphrase to be set with walletpassphrase call if wallet is encrypted.\n"}; @@ -78,9 +80,9 @@ bool HaveKey(const SigningProvider& wallet, const CKey& key) bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& wallet_name) { - if (request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { + if (URL_DECODE && request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { // wallet endpoint was used - wallet_name = urlDecode(request.URI.substr(WALLET_ENDPOINT_BASE.size())); + wallet_name = URL_DECODE(request.URI.substr(WALLET_ENDPOINT_BASE.size())); return true; } return false; @@ -143,8 +145,7 @@ static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& lo entry.pushKV("blockheight", wtx.m_confirm.block_height); entry.pushKV("blockindex", wtx.m_confirm.nIndex); int64_t block_time; - bool found_block = chain.findBlock(wtx.m_confirm.hashBlock, nullptr /* block */, &block_time); - CHECK_NONFATAL(found_block); + CHECK_NONFATAL(chain.findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(block_time))); entry.pushKV("blocktime", block_time); } else { entry.pushKV("trusted", wtx.IsTrusted(locked_chain)); @@ -571,6 +572,52 @@ static UniValue signmessage(const JSONRPCRequest& request) return signature; } +static CAmount GetReceived(interfaces::Chain::Lock& locked_chain, const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +{ + std::set<CTxDestination> address_set; + + if (by_label) { + // Get the set of addresses assigned to label + std::string label = LabelFromValue(params[0]); + address_set = wallet.GetLabelAddresses(label); + } else { + // Get the address + CTxDestination dest = DecodeDestination(params[0].get_str()); + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); + } + CScript script_pub_key = GetScriptForDestination(dest); + if (!wallet.IsMine(script_pub_key)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); + } + address_set.insert(dest); + } + + // Minimum confirmations + int min_depth = 1; + if (!params[1].isNull()) + min_depth = params[1].get_int(); + + // Tally + CAmount amount = 0; + for (const std::pair<const uint256, CWalletTx>& wtx_pair : wallet.mapWallet) { + const CWalletTx& wtx = wtx_pair.second; + if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx) || wtx.GetDepthInMainChain() < min_depth) { + continue; + } + + for (const CTxOut& txout : wtx.tx->vout) { + CTxDestination address; + if (ExtractDestination(txout.scriptPubKey, address) && wallet.IsMine(address) && address_set.count(address)) { + amount += txout.nValue; + } + } + } + + return amount; +} + + static UniValue getreceivedbyaddress(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -608,36 +655,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - // Bitcoin address - CTxDestination dest = DecodeDestination(request.params[0].get_str()); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); - } - CScript scriptPubKey = GetScriptForDestination(dest); - if (!pwallet->IsMine(scriptPubKey)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); - } - - // Minimum confirmations - int nMinDepth = 1; - if (!request.params[1].isNull()) - nMinDepth = request.params[1].get_int(); - - // Tally - CAmount nAmount = 0; - for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { - continue; - } - - for (const CTxOut& txout : wtx.tx->vout) - if (txout.scriptPubKey == scriptPubKey) - if (wtx.GetDepthInMainChain() >= nMinDepth) - nAmount += txout.nValue; - } - - return ValueFromAmount(nAmount); + return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ false)); } @@ -678,34 +696,7 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - // Minimum confirmations - int nMinDepth = 1; - if (!request.params[1].isNull()) - nMinDepth = request.params[1].get_int(); - - // Get the set of pub keys assigned to label - std::string label = LabelFromValue(request.params[0]); - std::set<CTxDestination> setAddress = pwallet->GetLabelAddresses(label); - - // Tally - CAmount nAmount = 0; - for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { - continue; - } - - for (const CTxOut& txout : wtx.tx->vout) - { - CTxDestination address; - if (ExtractDestination(txout.scriptPubKey, address) && pwallet->IsMine(address) && setAddress.count(address)) { - if (wtx.GetDepthInMainChain() >= nMinDepth) - nAmount += txout.nValue; - } - } - } - - return ValueFromAmount(nAmount); + return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ true)); } @@ -732,9 +723,9 @@ static UniValue getbalance(const JSONRPCRequest& request) RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " received for this wallet." }, RPCExamples{ - "\nThe total amount in the wallet with 1 or more confirmations\n" + "\nThe total amount in the wallet with 0 or more confirmations\n" + HelpExampleCli("getbalance", "") + - "\nThe total amount in the wallet at least 6 blocks confirmed\n" + "\nThe total amount in the wallet with at least 6 confirmations\n" + HelpExampleCli("getbalance", "\"*\" 6") + "\nAs a JSON-RPC call\n" + HelpExampleRpc("getbalance", "\"*\", 6") @@ -1399,8 +1390,8 @@ UniValue listtransactions(const JSONRPCRequest& request) "\nIf a label name is provided, this will return only incoming transactions paying to addresses with the specified label.\n" "\nReturns up to 'count' most recent transactions skipping the first 'from' transactions.\n", { - {"label", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If set, should be a valid label name to return only incoming transactions\n" - " with the specified label, or \"*\" to disable filtering and return all transactions."}, + {"label|dummy", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If set, should be a valid label name to return only incoming transactions\n" + "with the specified label, or \"*\" to disable filtering and return all transactions."}, {"count", RPCArg::Type::NUM, /* default */ "10", "The number of transactions to return"}, {"skip", RPCArg::Type::NUM, /* default */ "0", "The number of transactions to skip"}, {"include_watchonly", RPCArg::Type::BOOL, /* default */ "true for watch-only wallets, otherwise false", "Include transactions to watch-only addresses (see 'importaddress')"}, @@ -1578,8 +1569,9 @@ static UniValue listsinceblock(const JSONRPCRequest& request) uint256 blockId; if (!request.params[0].isNull() && !request.params[0].get_str().empty()) { blockId = ParseHashV(request.params[0], "blockhash"); - height = locked_chain->findFork(blockId, &altheight); - if (!height) { + height.emplace(); + altheight.emplace(); + if (!pwallet->chain().findCommonAncestor(blockId, pwallet->GetLastBlockHash(), /* ancestor out */ FoundBlock().height(*height), /* blockId out */ FoundBlock().height(*altheight))) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } } @@ -1598,8 +1590,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) bool include_removed = (request.params[3].isNull() || request.params[3].get_bool()); - const Optional<int> tip_height = locked_chain->getHeight(); - int depth = tip_height && height ? (1 + *tip_height - *height) : -1; + int depth = height ? pwallet->GetLastBlockHeight() + 1 - *height : -1; UniValue transactions(UniValue::VARR); @@ -1616,7 +1607,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) UniValue removed(UniValue::VARR); while (include_removed && altheight && *altheight > *height) { CBlock block; - if (!pwallet->chain().findBlock(blockId, &block) || block.IsNull()) { + if (!pwallet->chain().findBlock(blockId, FoundBlock().data(block)) || block.IsNull()) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); } for (const CTransactionRef& tx : block.vtx) { @@ -1631,8 +1622,8 @@ static UniValue listsinceblock(const JSONRPCRequest& request) --*altheight; } - int last_height = tip_height ? *tip_height + 1 - target_confirms : -1; - uint256 lastblock = last_height >= 0 ? locked_chain->getBlockHash(last_height) : uint256(); + uint256 lastblock; + CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(pwallet->GetLastBlockHash(), pwallet->GetLastBlockHeight() + 1 - target_confirms, FoundBlock().hash(lastblock))); UniValue ret(UniValue::VOBJ); ret.pushKV("transactions", transactions); @@ -1860,7 +1851,7 @@ static UniValue keypoolrefill(const JSONRPCRequest& request) }, }.Check(request); - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (pwallet->IsLegacy() && pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); } @@ -2324,7 +2315,8 @@ static UniValue settxfee(const JSONRPCRequest& request) } RPCHelpMan{"settxfee", - "\nSet the transaction fee per kB for this wallet. Overrides the global -paytxfee command line parameter.\n", + "\nSet the transaction fee per kB for this wallet. Overrides the global -paytxfee command line parameter.\n" + "Can be deactivated by passing 0 as the fee. In that case automatic fee selection will be used by default.\n", { {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The transaction fee in " + CURRENCY_UNIT + "/kB"}, }, @@ -2342,12 +2334,15 @@ static UniValue settxfee(const JSONRPCRequest& request) CAmount nAmount = AmountFromValue(request.params[0]); CFeeRate tx_fee_rate(nAmount, 1000); + CFeeRate max_tx_fee_rate(pwallet->m_default_max_tx_fee, 1000); if (tx_fee_rate == CFeeRate(0)) { // automatic selection } else if (tx_fee_rate < pwallet->chain().relayMinFee()) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than min relay tx fee (%s)", pwallet->chain().relayMinFee().ToString())); } else if (tx_fee_rate < pwallet->m_min_fee) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than wallet min fee (%s)", pwallet->m_min_fee.ToString())); + } else if (tx_fee_rate > max_tx_fee_rate) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be more than wallet max tx fee (%s)", max_tx_fee_rate.ToString())); } pwallet->m_pay_tx_fee = tx_fee_rate; @@ -2444,7 +2439,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) {RPCResult::Type::STR_AMOUNT, "unconfirmed_balance", "DEPRECATED. Identical to getbalances().mine.untrusted_pending"}, {RPCResult::Type::STR_AMOUNT, "immature_balance", "DEPRECATED. Identical to getbalances().mine.immature"}, {RPCResult::Type::NUM, "txcount", "the total number of transactions in the wallet"}, - {RPCResult::Type::NUM_TIME, "keypoololdest", "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool"}, + {RPCResult::Type::NUM_TIME, "keypoololdest", "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool. Legacy wallets only."}, {RPCResult::Type::NUM, "keypoolsize", "how many new keys are pre-generated (only counts external keys)"}, {RPCResult::Type::NUM, "keypoolsize_hd_internal", "how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)"}, {RPCResult::Type::NUM_TIME, "unlocked_until", "the " + UNIX_EPOCH_TIME + " until which the wallet is unlocked for transfers, or 0 if the wallet is locked"}, @@ -2457,6 +2452,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) {RPCResult::Type::NUM, "duration", "elapsed seconds since scan start"}, {RPCResult::Type::NUM, "progress", "scanning progress percentage [0.0, 1.0]"}, }}, + {RPCResult::Type::BOOL, "descriptors", "whether this wallet uses descriptors for scriptPubKey management"}, }}, }, RPCExamples{ @@ -2476,13 +2472,16 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) size_t kpExternalSize = pwallet->KeypoolCountExternalKeys(); const auto bal = pwallet->GetBalance(); + int64_t kp_oldest = pwallet->GetOldestKeyPoolTime(); obj.pushKV("walletname", pwallet->GetName()); obj.pushKV("walletversion", pwallet->GetVersion()); obj.pushKV("balance", ValueFromAmount(bal.m_mine_trusted)); obj.pushKV("unconfirmed_balance", ValueFromAmount(bal.m_mine_untrusted_pending)); obj.pushKV("immature_balance", ValueFromAmount(bal.m_mine_immature)); obj.pushKV("txcount", (int)pwallet->mapWallet.size()); - obj.pushKV("keypoololdest", pwallet->GetOldestKeyPoolTime()); + if (kp_oldest > 0) { + obj.pushKV("keypoololdest", kp_oldest); + } obj.pushKV("keypoolsize", (int64_t)kpExternalSize); LegacyScriptPubKeyMan* spk_man = pwallet->GetLegacyScriptPubKeyMan(); @@ -2510,6 +2509,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) } else { obj.pushKV("scanning", false); } + obj.pushKV("descriptors", pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); return obj; } @@ -2586,7 +2586,7 @@ static UniValue loadwallet(const JSONRPCRequest& request) RPCHelpMan{"loadwallet", "\nLoads a wallet from a wallet file or directory." "\nNote that all wallet command-line options used when starting bitcoind will be" - "\napplied to the new wallet (eg -zapwallettxes, upgradewallet, rescan, etc).\n", + "\napplied to the new wallet (eg -zapwallettxes, rescan, etc).\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet directory or .dat file."}, }, @@ -2706,6 +2706,7 @@ static UniValue createwallet(const JSONRPCRequest& request) {"blank", RPCArg::Type::BOOL, /* default */ "false", "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Encrypt the wallet with this passphrase."}, {"avoid_reuse", RPCArg::Type::BOOL, /* default */ "false", "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, + {"descriptors", RPCArg::Type::BOOL, /* default */ "false", "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -2742,6 +2743,9 @@ static UniValue createwallet(const JSONRPCRequest& request) if (!request.params[4].isNull() && request.params[4].get_bool()) { flags |= WALLET_FLAG_AVOID_REUSE; } + if (!request.params[5].isNull() && request.params[5].get_bool()) { + flags |= WALLET_FLAG_DESCRIPTORS; + } std::string error; std::shared_ptr<CWallet> wallet; @@ -3535,28 +3539,29 @@ UniValue rescanblockchain(const JSONRPCRequest& request) }, }.Check(request); - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } int start_height = 0; - uint256 start_block, stop_block; + Optional<int> stop_height; + uint256 start_block; { auto locked_chain = pwallet->chain().lock(); - Optional<int> tip_height = locked_chain->getHeight(); + LOCK(pwallet->cs_wallet); + int tip_height = pwallet->GetLastBlockHeight(); if (!request.params[0].isNull()) { start_height = request.params[0].get_int(); - if (start_height < 0 || !tip_height || start_height > *tip_height) { + if (start_height < 0 || start_height > tip_height) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid start_height"); } } - Optional<int> stop_height; if (!request.params[1].isNull()) { stop_height = request.params[1].get_int(); - if (*stop_height < 0 || !tip_height || *stop_height > *tip_height) { + if (*stop_height < 0 || *stop_height > tip_height) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stop_height"); } else if (*stop_height < start_height) { @@ -3565,25 +3570,15 @@ UniValue rescanblockchain(const JSONRPCRequest& request) } // We can't rescan beyond non-pruned blocks, stop and throw an error - if (locked_chain->findPruned(start_height, stop_height)) { + if (!pwallet->chain().hasBlocks(pwallet->GetLastBlockHash(), start_height, stop_height)) { throw JSONRPCError(RPC_MISC_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height."); } - if (tip_height) { - start_block = locked_chain->getBlockHash(start_height); - // If called with a stop_height, set the stop_height here to - // trigger a rescan to that height. - // If called without a stop height, leave stop_height as null here - // so rescan continues to the tip (even if the tip advances during - // rescan). - if (stop_height) { - stop_block = locked_chain->getBlockHash(*stop_height); - } - } + CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(pwallet->GetLastBlockHash(), start_height, FoundBlock().hash(start_block))); } CWallet::ScanResult result = - pwallet->ScanForWalletTransactions(start_block, stop_block, reserver, true /* fUpdate */); + pwallet->ScanForWalletTransactions(start_block, start_height, stop_height, reserver, true /* fUpdate */); switch (result.status) { case CWallet::ScanResult::SUCCESS: break; @@ -3829,7 +3824,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) ScriptPubKeyMan* spk_man = pwallet->GetScriptPubKeyMan(scriptPubKey); if (spk_man) { - if (const CKeyMetadata* meta = spk_man->GetMetadata(dest)) { + if (const std::unique_ptr<CKeyMetadata> meta = spk_man->GetMetadata(dest)) { ret.pushKV("timestamp", meta->nCreateTime); if (meta->has_key_origin) { ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); @@ -4021,7 +4016,7 @@ UniValue sethdseed(const JSONRPCRequest& request) // Do not do anything to non-HD wallets if (!pwallet->CanSupportFeature(FEATURE_HD)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Start with -upgradewallet in order to upgrade a non-HD wallet to HD"); + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Use the upgradewallet RPC in order to upgrade a non-HD wallet to HD"); } EnsureWalletIsUnlocked(pwallet); @@ -4245,6 +4240,45 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) return result; } +static UniValue upgradewallet(const JSONRPCRequest& request) +{ + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + CWallet* const pwallet = wallet.get(); + + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + RPCHelpMan{"upgradewallet", + "\nUpgrade the wallet. Upgrades to the latest version if no version number is specified\n" + "New keys may be generated and a new wallet backup will need to be made.", + { + {"version", RPCArg::Type::NUM, /* default */ strprintf("%d", FEATURE_LATEST), "The version number to upgrade to. Default is the latest wallet version"} + }, + RPCResults{}, + RPCExamples{ + HelpExampleCli("upgradewallet", "169900") + + HelpExampleRpc("upgradewallet", "169900") + } + }.Check(request); + + RPCTypeCheck(request.params, {UniValue::VNUM}, true); + + EnsureWalletIsUnlocked(pwallet); + + int version = 0; + if (!request.params[0].isNull()) { + version = request.params[0].get_int(); + } + + std::string error; + std::vector<std::string> warnings; + if (!pwallet->UpgradeWallet(version, error, warnings)) { + throw JSONRPCError(RPC_WALLET_ERROR, error); + } + return error; +} + UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp UniValue importprivkey(const JSONRPCRequest& request); @@ -4255,6 +4289,7 @@ UniValue importwallet(const JSONRPCRequest& request); UniValue importprunedfunds(const JSONRPCRequest& request); UniValue removeprunedfunds(const JSONRPCRequest& request); UniValue importmulti(const JSONRPCRequest& request); +UniValue importdescriptors(const JSONRPCRequest& request); void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers) { @@ -4268,7 +4303,7 @@ static const CRPCCommand commands[] = { "wallet", "addmultisigaddress", &addmultisigaddress, {"nrequired","keys","label","address_type"} }, { "wallet", "backupwallet", &backupwallet, {"destination"} }, { "wallet", "bumpfee", &bumpfee, {"txid", "options"} }, - { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse"} }, + { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse", "descriptors"} }, { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, { "wallet", "dumpwallet", &dumpwallet, {"filename"} }, { "wallet", "encryptwallet", &encryptwallet, {"passphrase"} }, @@ -4284,6 +4319,7 @@ static const CRPCCommand commands[] = { "wallet", "getbalances", &getbalances, {} }, { "wallet", "getwalletinfo", &getwalletinfo, {} }, { "wallet", "importaddress", &importaddress, {"address","label","rescan","p2sh"} }, + { "wallet", "importdescriptors", &importdescriptors, {"requests"} }, { "wallet", "importmulti", &importmulti, {"requests","options"} }, { "wallet", "importprivkey", &importprivkey, {"privkey","label","rescan"} }, { "wallet", "importprunedfunds", &importprunedfunds, {"rawtransaction","txoutproof"} }, @@ -4313,6 +4349,7 @@ static const CRPCCommand commands[] = { "wallet", "signmessage", &signmessage, {"address","message"} }, { "wallet", "signrawtransactionwithwallet", &signrawtransactionwithwallet, {"hexstring","prevtxs","sighashtype"} }, { "wallet", "unloadwallet", &unloadwallet, {"wallet_name"} }, + { "wallet", "upgradewallet", &upgradewallet, {"version"} }, { "wallet", "walletcreatefundedpsbt", &walletcreatefundedpsbt, {"inputs","outputs","locktime","options","bip32derivs"} }, { "wallet", "walletlock", &walletlock, {} }, { "wallet", "walletpassphrase", &walletpassphrase, {"passphrase","timeout"} }, diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index a7a29de9c6..8c149d455b 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index b96cb0aa1a..ecb95d599d 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,7 +20,7 @@ bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestinat // Generate a new key that is added to wallet CPubKey new_key; if (!GetKeyFromPool(new_key, type)) { - error = "Error: Keypool ran out, please call keypoolrefill first"; + error = _("Error: Keypool ran out, please call keypoolrefill first").translated; return false; } LearnRelatedScripts(new_key, type); @@ -568,7 +568,7 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb return TransactionError::OK; } -const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const +std::unique_ptr<CKeyMetadata> LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const { LOCK(cs_KeyStore); @@ -576,14 +576,14 @@ const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& des if (!key_id.IsNull()) { auto it = mapKeyMetadata.find(key_id); if (it != mapKeyMetadata.end()) { - return &it->second; + return MakeUnique<CKeyMetadata>(it->second); } } CScript scriptPubKey = GetScriptForDestination(dest); auto it = m_script_metadata.find(CScriptID(scriptPubKey)); if (it != m_script_metadata.end()) { - return &it->second; + return MakeUnique<CKeyMetadata>(it->second); } return nullptr; @@ -1497,3 +1497,668 @@ std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const } return set_address; } + +void LegacyScriptPubKeyMan::SetType(OutputType type, bool internal) {} + +bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) +{ + // Returns true if this descriptor supports getting new addresses. Conditions where we may be unable to fetch them (e.g. locked) are caught later + if (!CanGetAddresses(m_internal)) { + error = "No addresses available"; + return false; + } + { + LOCK(cs_desc_man); + assert(m_wallet_descriptor.descriptor->IsSingleType()); // This is a combo descriptor which should not be an active descriptor + if (type != m_address_type) { + throw std::runtime_error(std::string(__func__) + ": Types are inconsistent"); + } + + TopUp(); + + // Get the scriptPubKey from the descriptor + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + if (m_wallet_descriptor.range_end <= m_max_cached_index && !TopUp(1)) { + // We can't generate anymore keys + error = "Error: Keypool ran out, please call keypoolrefill first"; + return false; + } + if (!m_wallet_descriptor.descriptor->ExpandFromCache(m_wallet_descriptor.next_index, m_wallet_descriptor.cache, scripts_temp, out_keys)) { + // We can't generate anymore keys + error = "Error: Keypool ran out, please call keypoolrefill first"; + return false; + } + + Optional<OutputType> out_script_type = m_wallet_descriptor.descriptor->GetOutputType(); + if (out_script_type && out_script_type == type) { + ExtractDestination(scripts_temp[0], dest); + } else { + throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); + } + m_wallet_descriptor.next_index++; + WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); + return true; + } +} + +isminetype DescriptorScriptPubKeyMan::IsMine(const CScript& script) const +{ + LOCK(cs_desc_man); + if (m_map_script_pub_keys.count(script) > 0) { + return ISMINE_SPENDABLE; + } + return ISMINE_NO; +} + +bool DescriptorScriptPubKeyMan::CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys) +{ + LOCK(cs_desc_man); + if (!m_map_keys.empty()) { + return false; + } + + bool keyPass = m_map_crypted_keys.empty(); // Always pass when there are no encrypted keys + bool keyFail = false; + for (const auto& mi : m_map_crypted_keys) { + const CPubKey &pubkey = mi.second.first; + const std::vector<unsigned char> &crypted_secret = mi.second.second; + CKey key; + if (!DecryptKey(master_key, crypted_secret, pubkey, key)) { + keyFail = true; + break; + } + keyPass = true; + if (m_decryption_thoroughly_checked) + break; + } + if (keyPass && keyFail) { + LogPrintf("The wallet is probably corrupted: Some keys decrypt but not all.\n"); + throw std::runtime_error("Error unlocking wallet: some keys decrypt but not all. Your wallet file may be corrupt."); + } + if (keyFail || (!keyPass && !accept_no_keys)) { + return false; + } + m_decryption_thoroughly_checked = true; + return true; +} + +bool DescriptorScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) +{ + LOCK(cs_desc_man); + if (!m_map_crypted_keys.empty()) { + return false; + } + + for (const KeyMap::value_type& key_in : m_map_keys) + { + const CKey &key = key_in.second; + CPubKey pubkey = key.GetPubKey(); + CKeyingMaterial secret(key.begin(), key.end()); + std::vector<unsigned char> crypted_secret; + if (!EncryptSecret(master_key, secret, pubkey.GetHash(), crypted_secret)) { + return false; + } + m_map_crypted_keys[pubkey.GetID()] = make_pair(pubkey, crypted_secret); + batch->WriteCryptedDescriptorKey(GetID(), pubkey, crypted_secret); + } + m_map_keys.clear(); + return true; +} + +bool DescriptorScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) +{ + LOCK(cs_desc_man); + std::string error; + bool result = GetNewDestination(type, address, error); + index = m_wallet_descriptor.next_index - 1; + return result; +} + +void DescriptorScriptPubKeyMan::ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) +{ + LOCK(cs_desc_man); + // Only return when the index was the most recent + if (m_wallet_descriptor.next_index - 1 == index) { + m_wallet_descriptor.next_index--; + } + WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); + NotifyCanGetAddressesChanged(); +} + +std::map<CKeyID, CKey> DescriptorScriptPubKeyMan::GetKeys() const +{ + AssertLockHeld(cs_desc_man); + if (m_storage.HasEncryptionKeys() && !m_storage.IsLocked()) { + KeyMap keys; + for (auto key_pair : m_map_crypted_keys) { + const CPubKey& pubkey = key_pair.second.first; + const std::vector<unsigned char>& crypted_secret = key_pair.second.second; + CKey key; + DecryptKey(m_storage.GetEncryptionKey(), crypted_secret, pubkey, key); + keys[pubkey.GetID()] = key; + } + return keys; + } + return m_map_keys; +} + +bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) +{ + LOCK(cs_desc_man); + unsigned int target_size; + if (size > 0) { + target_size = size; + } else { + target_size = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); + } + + // Calculate the new range_end + int32_t new_range_end = std::max(m_wallet_descriptor.next_index + (int32_t)target_size, m_wallet_descriptor.range_end); + + // If the descriptor is not ranged, we actually just want to fill the first cache item + if (!m_wallet_descriptor.descriptor->IsRange()) { + new_range_end = 1; + m_wallet_descriptor.range_end = 1; + m_wallet_descriptor.range_start = 0; + } + + FlatSigningProvider provider; + provider.keys = GetKeys(); + + WalletBatch batch(m_storage.GetDatabase()); + uint256 id = GetID(); + for (int32_t i = m_max_cached_index + 1; i < new_range_end; ++i) { + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + DescriptorCache temp_cache; + // Maybe we have a cached xpub and we can expand from the cache first + if (!m_wallet_descriptor.descriptor->ExpandFromCache(i, m_wallet_descriptor.cache, scripts_temp, out_keys)) { + if (!m_wallet_descriptor.descriptor->Expand(i, provider, scripts_temp, out_keys, &temp_cache)) return false; + } + // Add all of the scriptPubKeys to the scriptPubKey set + for (const CScript& script : scripts_temp) { + m_map_script_pub_keys[script] = i; + } + for (const auto& pk_pair : out_keys.pubkeys) { + const CPubKey& pubkey = pk_pair.second; + if (m_map_pubkeys.count(pubkey) != 0) { + // We don't need to give an error here. + // It doesn't matter which of many valid indexes the pubkey has, we just need an index where we can derive it and it's private key + continue; + } + m_map_pubkeys[pubkey] = i; + } + // Write the cache + for (const auto& parent_xpub_pair : temp_cache.GetCachedParentExtPubKeys()) { + CExtPubKey xpub; + if (m_wallet_descriptor.cache.GetCachedParentExtPubKey(parent_xpub_pair.first, xpub)) { + if (xpub != parent_xpub_pair.second) { + throw std::runtime_error(std::string(__func__) + ": New cached parent xpub does not match already cached parent xpub"); + } + continue; + } + if (!batch.WriteDescriptorParentCache(parent_xpub_pair.second, id, parent_xpub_pair.first)) { + throw std::runtime_error(std::string(__func__) + ": writing cache item failed"); + } + m_wallet_descriptor.cache.CacheParentExtPubKey(parent_xpub_pair.first, parent_xpub_pair.second); + } + for (const auto& derived_xpub_map_pair : temp_cache.GetCachedDerivedExtPubKeys()) { + for (const auto& derived_xpub_pair : derived_xpub_map_pair.second) { + CExtPubKey xpub; + if (m_wallet_descriptor.cache.GetCachedDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, xpub)) { + if (xpub != derived_xpub_pair.second) { + throw std::runtime_error(std::string(__func__) + ": New cached derived xpub does not match already cached derived xpub"); + } + continue; + } + if (!batch.WriteDescriptorDerivedCache(derived_xpub_pair.second, id, derived_xpub_map_pair.first, derived_xpub_pair.first)) { + throw std::runtime_error(std::string(__func__) + ": writing cache item failed"); + } + m_wallet_descriptor.cache.CacheDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, derived_xpub_pair.second); + } + } + m_max_cached_index++; + } + m_wallet_descriptor.range_end = new_range_end; + batch.WriteDescriptor(GetID(), m_wallet_descriptor); + + // By this point, the cache size should be the size of the entire range + assert(m_wallet_descriptor.range_end - 1 == m_max_cached_index); + + NotifyCanGetAddressesChanged(); + return true; +} + +void DescriptorScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) +{ + LOCK(cs_desc_man); + if (IsMine(script)) { + int32_t index = m_map_script_pub_keys[script]; + if (index >= m_wallet_descriptor.next_index) { + WalletLogPrintf("%s: Detected a used keypool item at index %d, mark all keypool items up to this item as used\n", __func__, index); + m_wallet_descriptor.next_index = index + 1; + } + if (!TopUp()) { + WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); + } + } +} + +void DescriptorScriptPubKeyMan::AddDescriptorKey(const CKey& key, const CPubKey &pubkey) +{ + LOCK(cs_desc_man); + WalletBatch batch(m_storage.GetDatabase()); + if (!AddDescriptorKeyWithDB(batch, key, pubkey)) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor private key failed"); + } +} + +bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const CKey& key, const CPubKey &pubkey) +{ + AssertLockHeld(cs_desc_man); + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + + if (m_storage.HasEncryptionKeys()) { + if (m_storage.IsLocked()) { + return false; + } + + std::vector<unsigned char> crypted_secret; + CKeyingMaterial secret(key.begin(), key.end()); + if (!EncryptSecret(m_storage.GetEncryptionKey(), secret, pubkey.GetHash(), crypted_secret)) { + return false; + } + + m_map_crypted_keys[pubkey.GetID()] = make_pair(pubkey, crypted_secret); + return batch.WriteCryptedDescriptorKey(GetID(), pubkey, crypted_secret); + } else { + m_map_keys[pubkey.GetID()] = key; + return batch.WriteDescriptorKey(GetID(), pubkey, key.GetPrivKey()); + } +} + +bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key) +{ + LOCK(cs_desc_man); + assert(m_storage.IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + + // Ignore when there is already a descriptor + if (m_wallet_descriptor.descriptor) { + return false; + } + + int64_t creation_time = GetTime(); + + std::string xpub = EncodeExtPubKey(master_key.Neuter()); + + // Build descriptor string + std::string desc_prefix; + std::string desc_suffix = "/*)"; + switch (m_address_type) { + case OutputType::LEGACY: { + desc_prefix = "pkh(" + xpub + "/44'"; + break; + } + case OutputType::P2SH_SEGWIT: { + desc_prefix = "sh(wpkh(" + xpub + "/49'"; + desc_suffix += ")"; + break; + } + case OutputType::BECH32: { + desc_prefix = "wpkh(" + xpub + "/84'"; + break; + } + default: assert(false); + } + + // Mainnet derives at 0', testnet and regtest derive at 1' + if (Params().IsTestChain()) { + desc_prefix += "/1'"; + } else { + desc_prefix += "/0'"; + } + + std::string internal_path = m_internal ? "/1" : "/0"; + std::string desc_str = desc_prefix + "/0'" + internal_path + desc_suffix; + + // Make the descriptor + FlatSigningProvider keys; + std::string error; + std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, error, false); + WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); + m_wallet_descriptor = w_desc; + + // Store the master private key, and descriptor + WalletBatch batch(m_storage.GetDatabase()); + if (!AddDescriptorKeyWithDB(batch, master_key.key, master_key.key.GetPubKey())) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor master private key failed"); + } + if (!batch.WriteDescriptor(GetID(), m_wallet_descriptor)) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor failed"); + } + + // TopUp + TopUp(); + + m_storage.UnsetBlankWalletFlag(batch); + return true; +} + +bool DescriptorScriptPubKeyMan::IsHDEnabled() const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.descriptor->IsRange(); +} + +bool DescriptorScriptPubKeyMan::CanGetAddresses(bool internal) const +{ + // We can only give out addresses from descriptors that are single type (not combo), ranged, + // and either have cached keys or can generate more keys (ignoring encryption) + LOCK(cs_desc_man); + return m_wallet_descriptor.descriptor->IsSingleType() && + m_wallet_descriptor.descriptor->IsRange() && + (HavePrivateKeys() || m_wallet_descriptor.next_index < m_wallet_descriptor.range_end); +} + +bool DescriptorScriptPubKeyMan::HavePrivateKeys() const +{ + LOCK(cs_desc_man); + return m_map_keys.size() > 0 || m_map_crypted_keys.size() > 0; +} + +int64_t DescriptorScriptPubKeyMan::GetOldestKeyPoolTime() const +{ + // This is only used for getwalletinfo output and isn't relevant to descriptor wallets. + // The magic number 0 indicates that it shouldn't be displayed so that's what we return. + return 0; +} + +size_t DescriptorScriptPubKeyMan::KeypoolCountExternalKeys() const +{ + if (m_internal) { + return 0; + } + return GetKeyPoolSize(); +} + +unsigned int DescriptorScriptPubKeyMan::GetKeyPoolSize() const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.range_end - m_wallet_descriptor.next_index; +} + +int64_t DescriptorScriptPubKeyMan::GetTimeFirstKey() const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.creation_time; +} + +std::unique_ptr<FlatSigningProvider> DescriptorScriptPubKeyMan::GetSigningProvider(const CScript& script, bool include_private) const +{ + LOCK(cs_desc_man); + + // Find the index of the script + auto it = m_map_script_pub_keys.find(script); + if (it == m_map_script_pub_keys.end()) { + return nullptr; + } + int32_t index = it->second; + + return GetSigningProvider(index, include_private); +} + +std::unique_ptr<FlatSigningProvider> DescriptorScriptPubKeyMan::GetSigningProvider(const CPubKey& pubkey) const +{ + LOCK(cs_desc_man); + + // Find index of the pubkey + auto it = m_map_pubkeys.find(pubkey); + if (it == m_map_pubkeys.end()) { + return nullptr; + } + int32_t index = it->second; + + // Always try to get the signing provider with private keys. This function should only be called during signing anyways + return GetSigningProvider(index, true); +} + +std::unique_ptr<FlatSigningProvider> DescriptorScriptPubKeyMan::GetSigningProvider(int32_t index, bool include_private) const +{ + AssertLockHeld(cs_desc_man); + // Get the scripts, keys, and key origins for this script + std::unique_ptr<FlatSigningProvider> out_keys = MakeUnique<FlatSigningProvider>(); + std::vector<CScript> scripts_temp; + if (!m_wallet_descriptor.descriptor->ExpandFromCache(index, m_wallet_descriptor.cache, scripts_temp, *out_keys)) return nullptr; + + if (HavePrivateKeys() && include_private) { + FlatSigningProvider master_provider; + master_provider.keys = GetKeys(); + m_wallet_descriptor.descriptor->ExpandPrivate(index, master_provider, *out_keys); + } + + return out_keys; +} + +std::unique_ptr<SigningProvider> DescriptorScriptPubKeyMan::GetSolvingProvider(const CScript& script) const +{ + return GetSigningProvider(script, false); +} + +bool DescriptorScriptPubKeyMan::CanProvide(const CScript& script, SignatureData& sigdata) +{ + return IsMine(script); +} + +bool DescriptorScriptPubKeyMan::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const +{ + std::unique_ptr<FlatSigningProvider> keys = MakeUnique<FlatSigningProvider>(); + for (const auto& coin_pair : coins) { + std::unique_ptr<FlatSigningProvider> coin_keys = GetSigningProvider(coin_pair.second.out.scriptPubKey, true); + if (!coin_keys) { + continue; + } + *keys = Merge(*keys, *coin_keys); + } + + return ::SignTransaction(tx, keys.get(), coins, sighash, input_errors); +} + +SigningResult DescriptorScriptPubKeyMan::SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const +{ + std::unique_ptr<FlatSigningProvider> keys = GetSigningProvider(GetScriptForDestination(pkhash), true); + if (!keys) { + return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; + } + + CKeyID key_id(pkhash); + CKey key; + if (!keys->GetKey(key_id, key)) { + return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; + } + + if (!MessageSign(key, message, str_sig)) { + return SigningResult::SIGNING_FAILED; + } + return SigningResult::OK; +} + +TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, int sighash_type, bool sign, bool bip32derivs) const +{ + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + const CTxIn& txin = psbtx.tx->vin[i]; + PSBTInput& input = psbtx.inputs.at(i); + + if (PSBTInputSigned(input)) { + continue; + } + + // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. + if (!input.IsSane()) { + return TransactionError::INVALID_PSBT; + } + + // Get the Sighash type + if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { + return TransactionError::SIGHASH_MISMATCH; + } + + // Get the scriptPubKey to know which SigningProvider to use + CScript script; + if (!input.witness_utxo.IsNull()) { + script = input.witness_utxo.scriptPubKey; + } else if (input.non_witness_utxo) { + if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { + return TransactionError::MISSING_INPUTS; + } + script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; + } else { + // There's no UTXO so we can just skip this now + continue; + } + SignatureData sigdata; + input.FillSignatureData(sigdata); + + std::unique_ptr<FlatSigningProvider> keys = MakeUnique<FlatSigningProvider>(); + std::unique_ptr<FlatSigningProvider> script_keys = GetSigningProvider(script, sign); + if (script_keys) { + *keys = Merge(*keys, *script_keys); + } else { + // Maybe there are pubkeys listed that we can sign for + script_keys = MakeUnique<FlatSigningProvider>(); + for (const auto& pk_pair : input.hd_keypaths) { + const CPubKey& pubkey = pk_pair.first; + std::unique_ptr<FlatSigningProvider> pk_keys = GetSigningProvider(pubkey); + if (pk_keys) { + *keys = Merge(*keys, *pk_keys); + } + } + } + + SignPSBTInput(HidingSigningProvider(keys.get(), !sign, !bip32derivs), psbtx, i, sighash_type); + } + + // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change + for (unsigned int i = 0; i < psbtx.tx->vout.size(); ++i) { + std::unique_ptr<SigningProvider> keys = GetSolvingProvider(psbtx.tx->vout.at(i).scriptPubKey); + if (!keys) { + continue; + } + UpdatePSBTOutput(HidingSigningProvider(keys.get(), true, !bip32derivs), psbtx, i); + } + + return TransactionError::OK; +} + +std::unique_ptr<CKeyMetadata> DescriptorScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const +{ + std::unique_ptr<SigningProvider> provider = GetSigningProvider(GetScriptForDestination(dest)); + if (provider) { + KeyOriginInfo orig; + CKeyID key_id = GetKeyForDestination(*provider, dest); + if (provider->GetKeyOrigin(key_id, orig)) { + LOCK(cs_desc_man); + std::unique_ptr<CKeyMetadata> meta = MakeUnique<CKeyMetadata>(); + meta->key_origin = orig; + meta->has_key_origin = true; + meta->nCreateTime = m_wallet_descriptor.creation_time; + return meta; + } + } + return nullptr; +} + +uint256 DescriptorScriptPubKeyMan::GetID() const +{ + LOCK(cs_desc_man); + std::string desc_str = m_wallet_descriptor.descriptor->ToString(); + uint256 id; + CSHA256().Write((unsigned char*)desc_str.data(), desc_str.size()).Finalize(id.begin()); + return id; +} + +void DescriptorScriptPubKeyMan::SetType(OutputType type, bool internal) +{ + this->m_address_type = type; + this->m_internal = internal; +} + +void DescriptorScriptPubKeyMan::SetCache(const DescriptorCache& cache) +{ + LOCK(cs_desc_man); + m_wallet_descriptor.cache = cache; + for (int32_t i = m_wallet_descriptor.range_start; i < m_wallet_descriptor.range_end; ++i) { + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + if (!m_wallet_descriptor.descriptor->ExpandFromCache(i, m_wallet_descriptor.cache, scripts_temp, out_keys)) { + throw std::runtime_error("Error: Unable to expand wallet descriptor from cache"); + } + // Add all of the scriptPubKeys to the scriptPubKey set + for (const CScript& script : scripts_temp) { + if (m_map_script_pub_keys.count(script) != 0) { + throw std::runtime_error(strprintf("Error: Already loaded script at index %d as being at index %d", i, m_map_script_pub_keys[script])); + } + m_map_script_pub_keys[script] = i; + } + for (const auto& pk_pair : out_keys.pubkeys) { + const CPubKey& pubkey = pk_pair.second; + if (m_map_pubkeys.count(pubkey) != 0) { + // We don't need to give an error here. + // It doesn't matter which of many valid indexes the pubkey has, we just need an index where we can derive it and it's private key + continue; + } + m_map_pubkeys[pubkey] = i; + } + m_max_cached_index++; + } +} + +bool DescriptorScriptPubKeyMan::AddKey(const CKeyID& key_id, const CKey& key) +{ + LOCK(cs_desc_man); + m_map_keys[key_id] = key; + return true; +} + +bool DescriptorScriptPubKeyMan::AddCryptedKey(const CKeyID& key_id, const CPubKey& pubkey, const std::vector<unsigned char>& crypted_key) +{ + LOCK(cs_desc_man); + if (!m_map_keys.empty()) { + return false; + } + + m_map_crypted_keys[key_id] = make_pair(pubkey, crypted_key); + return true; +} + +bool DescriptorScriptPubKeyMan::HasWalletDescriptor(const WalletDescriptor& desc) const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.descriptor != nullptr && desc.descriptor != nullptr && m_wallet_descriptor.descriptor->ToString() == desc.descriptor->ToString(); +} + +void DescriptorScriptPubKeyMan::WriteDescriptor() +{ + LOCK(cs_desc_man); + WalletBatch batch(m_storage.GetDatabase()); + if (!batch.WriteDescriptor(GetID(), m_wallet_descriptor)) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor failed"); + } +} + +const WalletDescriptor DescriptorScriptPubKeyMan::GetWalletDescriptor() const +{ + return m_wallet_descriptor; +} + +const std::vector<CScript> DescriptorScriptPubKeyMan::GetScriptPubKeys() const +{ + LOCK(cs_desc_man); + std::vector<CScript> script_pub_keys; + script_pub_keys.reserve(m_map_script_pub_keys.size()); + + for (auto const& script_pub_key: m_map_script_pub_keys) { + script_pub_keys.push_back(script_pub_key.first); + } + return script_pub_keys; +} diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 8512eadf31..3117b13d35 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -6,6 +6,7 @@ #define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H #include <psbt.h> +#include <script/descriptor.h> #include <script/signingprovider.h> #include <script/standard.h> #include <util/error.h> @@ -204,7 +205,7 @@ public: virtual int64_t GetTimeFirstKey() const { return 0; } - virtual const CKeyMetadata* GetMetadata(const CTxDestination& dest) const { return nullptr; } + virtual std::unique_ptr<CKeyMetadata> GetMetadata(const CTxDestination& dest) const { return nullptr; } virtual std::unique_ptr<SigningProvider> GetSolvingProvider(const CScript& script) const { return nullptr; } @@ -222,6 +223,8 @@ public: virtual uint256 GetID() const { return uint256(); } + virtual void SetType(OutputType type, bool internal) {} + /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ template<typename... Params> void WalletLogPrintf(std::string fmt, Params... parameters) const { @@ -352,7 +355,7 @@ public: int64_t GetTimeFirstKey() const override; - const CKeyMetadata* GetMetadata(const CTxDestination& dest) const override; + std::unique_ptr<CKeyMetadata> GetMetadata(const CTxDestination& dest) const override; bool CanGetAddresses(bool internal = false) const override; @@ -366,6 +369,8 @@ public: uint256 GetID() const override; + void SetType(OutputType type, bool internal) override; + // Map from Key ID to key metadata. std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_KeyStore); @@ -477,4 +482,111 @@ public: bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override { return m_spk_man.GetKeyOrigin(keyid, info); } }; +class DescriptorScriptPubKeyMan : public ScriptPubKeyMan +{ +private: + WalletDescriptor m_wallet_descriptor GUARDED_BY(cs_desc_man); + + using ScriptPubKeyMap = std::map<CScript, int32_t>; // Map of scripts to descriptor range index + using PubKeyMap = std::map<CPubKey, int32_t>; // Map of pubkeys involved in scripts to descriptor range index + using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; + using KeyMap = std::map<CKeyID, CKey>; + + ScriptPubKeyMap m_map_script_pub_keys GUARDED_BY(cs_desc_man); + PubKeyMap m_map_pubkeys GUARDED_BY(cs_desc_man); + int32_t m_max_cached_index = -1; + + OutputType m_address_type; + bool m_internal; + + KeyMap m_map_keys GUARDED_BY(cs_desc_man); + CryptedKeyMap m_map_crypted_keys GUARDED_BY(cs_desc_man); + + bool SetCrypted(); + + //! keeps track of whether Unlock has run a thorough check before + bool m_decryption_thoroughly_checked = false; + + bool AddDescriptorKeyWithDB(WalletBatch& batch, const CKey& key, const CPubKey &pubkey); + + KeyMap GetKeys() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + + // Fetch the SigningProvider for the given script and optionally include private keys + std::unique_ptr<FlatSigningProvider> GetSigningProvider(const CScript& script, bool include_private = false) const; + // Fetch the SigningProvider for the given pubkey and always include private keys. This should only be called by signing code. + std::unique_ptr<FlatSigningProvider> GetSigningProvider(const CPubKey& pubkey) const; + // Fetch the SigningProvider for a given index and optionally include private keys. Called by the above functions. + std::unique_ptr<FlatSigningProvider> GetSigningProvider(int32_t index, bool include_private = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + +public: + DescriptorScriptPubKeyMan(WalletStorage& storage, WalletDescriptor& descriptor) + : ScriptPubKeyMan(storage), + m_wallet_descriptor(descriptor) + {} + DescriptorScriptPubKeyMan(WalletStorage& storage, OutputType address_type, bool internal) + : ScriptPubKeyMan(storage), + m_address_type(address_type), m_internal(internal) + {} + + mutable RecursiveMutex cs_desc_man; + + bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; + isminetype IsMine(const CScript& script) const override; + + bool CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys = false) override; + bool Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) override; + + bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) override; + void ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) override; + + // Tops up the descriptor cache and m_map_script_pub_keys. The cache is stored in the wallet file + // and is used to expand the descriptor in GetNewDestination. DescriptorScriptPubKeyMan relies + // more on ephemeral data than LegacyScriptPubKeyMan. For wallets using unhardened derivation + // (with or without private keys), the "keypool" is a single xpub. + bool TopUp(unsigned int size = 0) override; + + void MarkUnusedAddresses(const CScript& script) override; + + bool IsHDEnabled() const override; + + //! Setup descriptors based on the given CExtkey + bool SetupDescriptorGeneration(const CExtKey& master_key); + + bool HavePrivateKeys() const override; + + int64_t GetOldestKeyPoolTime() const override; + size_t KeypoolCountExternalKeys() const override; + unsigned int GetKeyPoolSize() const override; + + int64_t GetTimeFirstKey() const override; + + std::unique_ptr<CKeyMetadata> GetMetadata(const CTxDestination& dest) const override; + + bool CanGetAddresses(bool internal = false) const override; + + std::unique_ptr<SigningProvider> GetSolvingProvider(const CScript& script) const override; + + bool CanProvide(const CScript& script, SignatureData& sigdata) override; + + bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const override; + SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; + TransactionError FillPSBT(PartiallySignedTransaction& psbt, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false) const override; + + uint256 GetID() const override; + + void SetType(OutputType type, bool internal) override; + + void SetCache(const DescriptorCache& cache); + + bool AddKey(const CKeyID& key_id, const CKey& key); + bool AddCryptedKey(const CKeyID& key_id, const CPubKey& pubkey, const std::vector<unsigned char>& crypted_key); + + bool HasWalletDescriptor(const WalletDescriptor& desc) const; + void AddDescriptorKey(const CKey& key, const CPubKey &pubkey); + void WriteDescriptor(); + + const WalletDescriptor GetWalletDescriptor() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + const std::vector<CScript> GetScriptPubKeys() const; +}; + #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 21d57cb898..66f4542cf9 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -1,16 +1,16 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <node/context.h> -#include <wallet/wallet.h> -#include <wallet/coinselection.h> -#include <wallet/coincontrol.h> #include <amount.h> +#include <node/context.h> #include <primitives/transaction.h> #include <random.h> #include <test/util/setup_common.h> +#include <wallet/coincontrol.h> +#include <wallet/coinselection.h> #include <wallet/test/wallet_test_fixture.h> +#include <wallet/wallet.h> #include <boost/test/unit_test.hpp> #include <random> @@ -176,8 +176,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) selection.clear(); // Select 5 Cent - add_coin(3 * CENT, 3, actual_selection); - add_coin(2 * CENT, 2, actual_selection); + add_coin(4 * CENT, 4, actual_selection); + add_coin(1 * CENT, 1, actual_selection); BOOST_CHECK(SelectCoinsBnB(GroupCoins(utxo_pool), 5 * CENT, 0.5 * CENT, selection, value_ret, not_input_fees)); BOOST_CHECK(equal_sets(selection, actual_selection)); BOOST_CHECK_EQUAL(value_ret, 5 * CENT); @@ -204,9 +204,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) // Select 10 Cent add_coin(5 * CENT, 5, utxo_pool); + add_coin(5 * CENT, 5, actual_selection); add_coin(4 * CENT, 4, actual_selection); - add_coin(3 * CENT, 3, actual_selection); - add_coin(2 * CENT, 2, actual_selection); add_coin(1 * CENT, 1, actual_selection); BOOST_CHECK(SelectCoinsBnB(GroupCoins(utxo_pool), 10 * CENT, 0.5 * CENT, selection, value_ret, not_input_fees)); BOOST_CHECK(equal_sets(selection, actual_selection)); diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 8b7b7af21d..35860577cd 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index b9e714946d..7ba3148ff3 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index 81d8a60b8a..a294935b64 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index a487e9e2e0..2bdeb89f3c 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -46,7 +46,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); - // Verify ScanForWalletTransactions accommodates a null start block. + // Verify ScanForWalletTransactions fails to read an unknown start block. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); { @@ -54,10 +54,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions({} /* start_block */, {} /* stop_block */, reserver, false /* update */); - BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); + CWallet::ScanResult result = wallet.ScanForWalletTransactions({} /* start_block */, 0 /* start_height */, {} /* max_height */, reserver, false /* update */); + BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK(result.last_scanned_block.IsNull()); BOOST_CHECK(!result.last_scanned_height); @@ -73,9 +73,9 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -96,9 +96,9 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, oldTip->GetBlockHash()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -118,9 +118,9 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, newTip->GetBlockHash()); BOOST_CHECK(result.last_scanned_block.IsNull()); @@ -152,6 +152,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); wallet->SetupLegacyScriptPubKeyMan(); + WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); AddWallet(wallet); UniValue keys; keys.setArray(); @@ -216,15 +217,19 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); - spk_man->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; - spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + { + auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); + spk_man->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; + spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + AddWallet(wallet); + wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } JSONRPCRequest request; request.params.setArray(); request.params.push_back(backup_file); - AddWallet(wallet); + ::dumpwallet(request); RemoveWallet(wallet); } @@ -233,16 +238,17 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // were scanned, and no prior blocks were scanned. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); JSONRPCRequest request; request.params.setArray(); request.params.push_back(backup_file); AddWallet(wallet); + wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); ::importwallet(request); RemoveWallet(wallet); - LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.size(), 3U); BOOST_CHECK_EQUAL(m_coinbase_txns.size(), 103U); for (size_t i = 0; i < m_coinbase_txns.size(); ++i) { @@ -454,15 +460,15 @@ public: CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); { - LOCK(wallet->cs_wallet); + LOCK2(::cs_main, wallet->cs_wallet); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); - WalletRescanReserver reserver(wallet.get()); + WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(::ChainActive().Genesis()->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(::ChainActive().Genesis()->GetBlockHash(), 0 /* start_height */, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK_EQUAL(result.last_scanned_block, ::ChainActive().Tip()->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, ::ChainActive().Height()); @@ -631,4 +637,25 @@ BOOST_FIXTURE_TEST_CASE(dummy_input_size_test, TestChain100Setup) BOOST_CHECK_EQUAL(CalculateNestedKeyhashInputSize(true), DUMMY_NESTED_P2WPKH_INPUT_SIZE); } +bool malformed_descriptor(std::ios_base::failure e) +{ + std::string s(e.what()); + return s.find("Missing checksum") != std::string::npos; +} + +BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) +{ + std::vector<unsigned char> malformed_record; + CVectorWriter vw(0, 0, malformed_record, 0); + vw << std::string("notadescriptor"); + vw << (uint64_t)0; + vw << (int32_t)0; + vw << (int32_t)0; + vw << (int32_t)1; + + VectorReader vr(0, 0, malformed_record, 0); + WalletDescriptor w_desc; + BOOST_CHECK_EXCEPTION(vr >> w_desc, std::ios_base::failure, malformed_descriptor); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 45f5542cad..b6f25de64e 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -22,6 +22,7 @@ #include <script/script.h> #include <script/signingprovider.h> #include <util/bip32.h> +#include <util/check.h> #include <util/error.h> #include <util/fees.h> #include <util/moneystr.h> @@ -35,6 +36,8 @@ #include <boost/algorithm/string/replace.hpp> +using interfaces::FoundBlock; + const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS{ {WALLET_FLAG_AVOID_REUSE, "You need to rescan the blockchain in order to correctly mark used " @@ -225,10 +228,14 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& // Set a seed for the wallet { LOCK(wallet->cs_wallet); - for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { - if (!spk_man->SetupGeneration()) { - error = "Unable to generate initial keys"; - return WalletCreationStatus::CREATION_FAILED; + if (wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + wallet->SetupDescriptorScriptPubKeyMans(); + } else { + for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { + if (!spk_man->SetupGeneration()) { + error = "Unable to generate initial keys"; + return WalletCreationStatus::CREATION_FAILED; + } } } } @@ -585,8 +592,11 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) Lock(); Unlock(strWalletPassphrase); - // if we are using HD, replace the HD seed with a new one - if (auto spk_man = GetLegacyScriptPubKeyMan()) { + // If we are using descriptors, make new descriptors with a new seed + if (IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)) { + SetupDescriptorScriptPubKeyMans(); + } else if (auto spk_man = GetLegacyScriptPubKeyMan()) { + // if we are using HD, replace the HD seed with a new one if (spk_man->IsHDEnabled()) { if (!spk_man->SetupGeneration(true)) { return false; @@ -744,22 +754,29 @@ bool CWallet::IsSpentKey(const uint256& hash, unsigned int n) const const CWalletTx* srctx = GetWalletTx(hash); if (srctx) { assert(srctx->tx->vout.size() > n); - LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); - // When descriptor wallets arrive, these additional checks are - // likely superfluous and can be optimized out - assert(spk_man != nullptr); - for (const auto& keyid : GetAffectedKeys(srctx->tx->vout[n].scriptPubKey, *spk_man)) { - WitnessV0KeyHash wpkh_dest(keyid); - if (GetDestData(wpkh_dest, "used", nullptr)) { - return true; - } - ScriptHash sh_wpkh_dest(GetScriptForDestination(wpkh_dest)); - if (GetDestData(sh_wpkh_dest, "used", nullptr)) { - return true; - } - PKHash pkh_dest(keyid); - if (GetDestData(pkh_dest, "used", nullptr)) { - return true; + CTxDestination dest; + if (!ExtractDestination(srctx->tx->vout[n].scriptPubKey, dest)) { + return false; + } + if (GetDestData(dest, "used", nullptr)) { + return true; + } + if (IsLegacy()) { + LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); + assert(spk_man != nullptr); + for (const auto& keyid : GetAffectedKeys(srctx->tx->vout[n].scriptPubKey, *spk_man)) { + WitnessV0KeyHash wpkh_dest(keyid); + if (GetDestData(wpkh_dest, "used", nullptr)) { + return true; + } + ScriptHash sh_wpkh_dest(GetScriptForDestination(wpkh_dest)); + if (GetDestData(sh_wpkh_dest, "used", nullptr)) { + return true; + } + PKHash pkh_dest(keyid); + if (GetDestData(pkh_dest, "used", nullptr)) { + return true; + } } } } @@ -1328,9 +1345,10 @@ CAmount CWallet::GetChange(const CTransaction& tx) const bool CWallet::IsHDEnabled() const { + // All Active ScriptPubKeyMans must be HD for this to be true bool result = true; - for (const auto& spk_man_pair : m_spk_managers) { - result &= spk_man_pair.second->IsHDEnabled(); + for (const auto& spk_man : GetActiveScriptPubKeyMans()) { + result &= spk_man->IsHDEnabled(); } return result; } @@ -1594,22 +1612,17 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r // Find starting block. May be null if nCreateTime is greater than the // highest blockchain timestamp, in which case there is nothing that needs // to be scanned. + int start_height = 0; uint256 start_block; - { - auto locked_chain = chain().lock(); - const Optional<int> start_height = locked_chain->findFirstBlockWithTimeAndHeight(startTime - TIMESTAMP_WINDOW, 0, &start_block); - const Optional<int> tip_height = locked_chain->getHeight(); - WalletLogPrintf("%s: Rescanning last %i blocks\n", __func__, tip_height && start_height ? *tip_height - *start_height + 1 : 0); - } + bool start = chain().findFirstBlockWithTimeAndHeight(startTime - TIMESTAMP_WINDOW, 0, FoundBlock().hash(start_block).height(start_height)); + WalletLogPrintf("%s: Rescanning last %i blocks\n", __func__, start ? WITH_LOCK(cs_wallet, return GetLastBlockHeight()) - start_height + 1 : 0); - if (!start_block.IsNull()) { + if (start) { // TODO: this should take into account failure by ScanResult::USER_ABORT - ScanResult result = ScanForWalletTransactions(start_block, {} /* stop_block */, reserver, update); + ScanResult result = ScanForWalletTransactions(start_block, start_height, {} /* max_height */, reserver, update); if (result.status == ScanResult::FAILURE) { int64_t time_max; - if (!chain().findBlock(result.last_failed_block, nullptr /* block */, nullptr /* time */, &time_max)) { - throw std::logic_error("ScanForWalletTransactions returned invalid block hash"); - } + CHECK_NONFATAL(chain().findBlock(result.last_failed_block, FoundBlock().maxTime(time_max))); return time_max + TIMESTAMP_WINDOW + 1; } } @@ -1623,9 +1636,9 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r * * @param[in] start_block Scan starting block. If block is not on the active * chain, the scan will return SUCCESS immediately. - * @param[in] stop_block Scan ending block. If block is not on the active - * chain, the scan will continue until it reaches the - * chain tip. + * @param[in] start_height Height of start_block + * @param[in] max_height Optional max scanning height. If unset there is + * no maximum and scanning can continue to the tip * * @return ScanResult returning scan information and indicating success or * failure. Return status will be set to SUCCESS if scan was @@ -1637,7 +1650,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r * the main chain after to the addition of any new keys you want to detect * transactions for. */ -CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, const uint256& stop_block, const WalletRescanReserver& reserver, bool fUpdate) +CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, Optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate) { int64_t nNow = GetTime(); int64_t start_time = GetTimeMillis(); @@ -1651,36 +1664,32 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc fAbortRescan = false; ShowProgress(strprintf("%s " + _("Rescanning...").translated, GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup - uint256 tip_hash; - // The way the 'block_height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. - Optional<int> block_height = MakeOptional(false, int()); - double progress_begin; - double progress_end; - { - auto locked_chain = chain().lock(); - if (Optional<int> tip_height = locked_chain->getHeight()) { - tip_hash = locked_chain->getBlockHash(*tip_height); - } - block_height = locked_chain->getBlockHeight(block_hash); - progress_begin = chain().guessVerificationProgress(block_hash); - progress_end = chain().guessVerificationProgress(stop_block.IsNull() ? tip_hash : stop_block); - } + uint256 tip_hash = WITH_LOCK(cs_wallet, return GetLastBlockHash()); + uint256 end_hash = tip_hash; + if (max_height) chain().findAncestorByHeight(tip_hash, *max_height, FoundBlock().hash(end_hash)); + double progress_begin = chain().guessVerificationProgress(block_hash); + double progress_end = chain().guessVerificationProgress(end_hash); double progress_current = progress_begin; - while (block_height && !fAbortRescan && !chain().shutdownRequested()) { + int block_height = start_height; + while (!fAbortRescan && !chain().shutdownRequested()) { m_scanning_progress = (progress_current - progress_begin) / (progress_end - progress_begin); - if (*block_height % 100 == 0 && progress_end - progress_begin > 0.0) { + if (block_height % 100 == 0 && progress_end - progress_begin > 0.0) { ShowProgress(strprintf("%s " + _("Rescanning...").translated, GetDisplayName()), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); } if (GetTime() >= nNow + 60) { nNow = GetTime(); - WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", *block_height, progress_current); + WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } CBlock block; - if (chain().findBlock(block_hash, &block) && !block.IsNull()) { + bool next_block; + uint256 next_block_hash; + bool reorg = false; + if (chain().findBlock(block_hash, FoundBlock().data(block)) && !block.IsNull()) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - if (!locked_chain->getBlockHeight(block_hash)) { + next_block = chain().findNextBlock(block_hash, block_height, FoundBlock().hash(next_block_hash), &reorg); + if (reorg) { // Abort scan if current block is no longer active, to prevent // marking transactions as coming from the wrong block. // TODO: This should return success instead of failure, see @@ -1690,37 +1699,38 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *block_height, block_hash, posInBlock); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, block_height, block_hash, posInBlock); SyncTransaction(block.vtx[posInBlock], confirm, fUpdate); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; - result.last_scanned_height = *block_height; + result.last_scanned_height = block_height; } else { // could not scan block, keep scanning but record this block as the most recent failure result.last_failed_block = block_hash; result.status = ScanResult::FAILURE; + next_block = chain().findNextBlock(block_hash, block_height, FoundBlock().hash(next_block_hash), &reorg); } - if (block_hash == stop_block) { + if (max_height && block_height >= *max_height) { break; } { auto locked_chain = chain().lock(); - Optional<int> tip_height = locked_chain->getHeight(); - if (!tip_height || *tip_height <= block_height || !locked_chain->getBlockHeight(block_hash)) { + if (!next_block || reorg) { // break successfully when rescan has reached the tip, or // previous block is no longer on the chain due to a reorg break; } // increment block and verification progress - block_hash = locked_chain->getBlockHash(++*block_height); + block_hash = next_block_hash; + ++block_height; progress_current = chain().guessVerificationProgress(block_hash); // handle updated tip hash const uint256 prev_tip_hash = tip_hash; - tip_hash = locked_chain->getBlockHash(*tip_height); - if (stop_block.IsNull() && prev_tip_hash != tip_hash) { + tip_hash = WITH_LOCK(cs_wallet, return GetLastBlockHash()); + if (!max_height && prev_tip_hash != tip_hash) { // in case the tip has changed, update progress max progress_end = chain().guessVerificationProgress(tip_hash); } @@ -1728,10 +1738,10 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc } ShowProgress(strprintf("%s " + _("Rescanning...").translated, GetDisplayName()), 100); // hide progress dialog in GUI if (block_height && fAbortRescan) { - WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", *block_height, progress_current); + WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; } else if (block_height && chain().shutdownRequested()) { - WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", *block_height, progress_current); + WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; } else { WalletLogPrintf("Rescan completed in %15dms\n", GetTimeMillis() - start_time); @@ -2377,6 +2387,13 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm ++it; } + unsigned int limit_ancestor_count = 0; + unsigned int limit_descendant_count = 0; + chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); + size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); + size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); + bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); + // form groups from remaining coins; note that preset coins will not // automatically have their associated (same address) coins included if (coin_control.m_avoid_partial_spends && vCoins.size() > OUTPUT_GROUP_MAX_ENTRIES) { @@ -2385,14 +2402,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // explicitly shuffling the outputs before processing Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); } - std::vector<OutputGroup> groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends); - - unsigned int limit_ancestor_count; - unsigned int limit_descendant_count; - chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); - size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); - size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); - bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); + std::vector<OutputGroup> groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends, max_ancestors); bool res = value_to_select <= 0 || SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || @@ -2432,11 +2442,17 @@ bool CWallet::SignTransaction(CMutableTransaction& tx) const bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const { - // Sign the tx with ScriptPubKeyMans - // Because each ScriptPubKeyMan can sign more than one input, we need to keep track of each ScriptPubKeyMan that has signed this transaction. - // Each iteration, we may sign more txins than the txin that is specified in that iteration. - // We assume that each input is signed by only one ScriptPubKeyMan. - std::set<uint256> visited_spk_mans; + // Try to sign with all ScriptPubKeyMans + for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { + // spk_man->SignTransaction will return true if the transaction is complete, + // so we can exit early and return true if that happens + if (spk_man->SignTransaction(tx, coins, sighash, input_errors)) { + return true; + } + } + + // At this point, one input was not fully signed otherwise we would have exited already + // Find that input and figure out what went wrong. for (unsigned int i = 0; i < tx.vin.size(); i++) { // Get the prevout CTxIn& txin = tx.vin[i]; @@ -2448,33 +2464,10 @@ bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, // Check if this input is complete SignatureData sigdata = DataFromTransaction(tx, i, coin->second.out); - if (sigdata.complete) { - continue; - } - - // Input needs to be signed, find the right ScriptPubKeyMan - std::set<ScriptPubKeyMan*> spk_mans = GetScriptPubKeyMans(coin->second.out.scriptPubKey, sigdata); - if (spk_mans.size() == 0) { + if (!sigdata.complete) { input_errors[i] = "Unable to sign input, missing keys"; continue; } - - for (auto& spk_man : spk_mans) { - // If we've already been signed by this spk_man, skip it - if (visited_spk_mans.count(spk_man->GetID()) > 0) { - continue; - } - - // Sign the tx. - // spk_man->SignTransaction will return true if the transaction is complete, - // so we can exit early and return true if that happens. - if (spk_man->SignTransaction(tx, coins, sighash, input_errors)) { - return true; - } - - // Add this spk_man to visited_spk_mans so we can skip it later - visited_spk_mans.insert(spk_man->GetID()); - } } return false; } @@ -2510,52 +2503,10 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp } // Fill in information from ScriptPubKeyMans - // Because each ScriptPubKeyMan may be able to fill more than one input, we need to keep track of each ScriptPubKeyMan that has filled this psbt. - // Each iteration, we may fill more inputs than the input that is specified in that iteration. - // We assume that each input is filled by only one ScriptPubKeyMan - std::set<uint256> visited_spk_mans; - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - const CTxIn& txin = psbtx.tx->vin[i]; - PSBTInput& input = psbtx.inputs.at(i); - - if (PSBTInputSigned(input)) { - continue; - } - - // Get the scriptPubKey to know which ScriptPubKeyMan to use - CScript script; - if (!input.witness_utxo.IsNull()) { - script = input.witness_utxo.scriptPubKey; - } else if (input.non_witness_utxo) { - if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { - return TransactionError::MISSING_INPUTS; - } - script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; - } else { - // There's no UTXO so we can just skip this now - continue; - } - SignatureData sigdata; - input.FillSignatureData(sigdata); - std::set<ScriptPubKeyMan*> spk_mans = GetScriptPubKeyMans(script, sigdata); - if (spk_mans.size() == 0) { - continue; - } - - for (auto& spk_man : spk_mans) { - // If we've already been signed by this spk_man, skip it - if (visited_spk_mans.count(spk_man->GetID()) > 0) { - continue; - } - - // Fill in the information from the spk_man - TransactionError res = spk_man->FillPSBT(psbtx, sighash_type, sign, bip32derivs); - if (res != TransactionError::OK) { - return res; - } - - // Add this spk_man to visited_spk_mans so we can skip it later - visited_spk_mans.insert(spk_man->GetID()); + for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { + TransactionError res = spk_man->FillPSBT(psbtx, sighash_type, sign, bip32derivs); + if (res != TransactionError::OK) { + return res; } } @@ -2631,13 +2582,15 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC return true; } -static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain) +static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, const uint256& block_hash) { if (chain.isInitialBlockDownload()) { return false; } constexpr int64_t MAX_ANTI_FEE_SNIPING_TIP_AGE = 8 * 60 * 60; // in seconds - if (locked_chain.getBlockTime(*locked_chain.getHeight()) < (GetTime() - MAX_ANTI_FEE_SNIPING_TIP_AGE)) { + int64_t block_time; + CHECK_NONFATAL(chain.findBlock(block_hash, FoundBlock().time(block_time))); + if (block_time < (GetTime() - MAX_ANTI_FEE_SNIPING_TIP_AGE)) { return false; } return true; @@ -2647,9 +2600,8 @@ static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, interfaces::Cha * Return a height-based locktime for new transactions (uses the height of the * current chain tip unless we are not synced with the current chain */ -static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain) +static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uint256& block_hash, int block_height) { - uint32_t const height = locked_chain.getHeight().get_value_or(-1); uint32_t locktime; // Discourage fee sniping. // @@ -2671,8 +2623,8 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interface // enough, that fee sniping isn't a problem yet, but by implementing a fix // now we ensure code won't be written that makes assumptions about // nLockTime that preclude a fix later. - if (IsCurrentForAntiFeeSniping(chain, locked_chain)) { - locktime = height; + if (IsCurrentForAntiFeeSniping(chain, block_hash)) { + locktime = block_height; // Secondly occasionally randomly pick a nLockTime even further back, so // that transactions that are delayed after signing for whatever reason, @@ -2686,7 +2638,6 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interface // unique "nLockTime fingerprint", set nLockTime to a constant. locktime = 0; } - assert(locktime <= height); assert(locktime < LOCKTIME_THRESHOLD); return locktime; } @@ -2746,9 +2697,6 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } CMutableTransaction txNew; - - txNew.nLockTime = GetLocktimeForNewTransaction(chain(), locked_chain); - FeeCalculation feeCalc; CAmount nFeeNeeded; int nBytes; @@ -2756,6 +2704,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std std::set<CInputCoin> setCoins; auto locked_chain = chain().lock(); LOCK(cs_wallet); + txNew.nLockTime = GetLocktimeForNewTransaction(chain(), GetLastBlockHash(), GetLastBlockHeight()); { std::vector<COutput> vAvailableCoins; AvailableCoins(*locked_chain, vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); @@ -2777,20 +2726,14 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // rediscover unknown transactions that were written with keys of ours to recover // post-backup change. - // Reserve a new key pair from key pool - if (!CanGetAddresses(true)) { - strFailReason = _("Can't generate a change-address key. No keys in the internal keypool and can't generate any keys.").translated; - return false; - } + // Reserve a new key pair from key pool. If it fails, provide a dummy + // destination in case we don't need change. CTxDestination dest; - bool ret = reservedest.GetReservedDestination(dest, true); - if (!ret) - { - strFailReason = "Keypool ran out, please call keypoolrefill first"; - return false; + if (!reservedest.GetReservedDestination(dest, true)) { + strFailReason = _("Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.").translated; } - scriptChange = GetScriptForDestination(dest); + assert(!dest.empty() || scriptChange.empty()); } CTxOut change_prototype_txout(0, scriptChange); coin_selection_params.change_output_size = GetSerializeSize(change_prototype_txout); @@ -3006,6 +2949,11 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std coin_selection_params.use_bnb = false; continue; } + + // Give up if change keypool ran out and we failed to find a solution without change: + if (scriptChange.empty() && nChangePosInOut != -1) { + return false; + } } // Shuffle selected coins and fill in final vin @@ -3286,6 +3234,8 @@ bool CWallet::GetNewDestination(const OutputType type, const std::string label, if (spk_man) { spk_man->TopUp(); result = spk_man->GetNewDestination(type, dest, error); + } else { + error = strprintf("Error: No %s addresses available.", FormatOutputType(type)); } if (result) { SetAddressBook(dest, label, "receive"); @@ -3301,7 +3251,7 @@ bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& des ReserveDestination reservedest(this, type); if (!reservedest.GetReservedDestination(dest, true)) { - error = "Error: Keypool ran out, please call keypoolrefill first"; + error = _("Error: Keypool ran out, please call keypoolrefill first").translated; return false; } @@ -3576,12 +3526,13 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C } // map in which we'll infer heights of other keys - const Optional<int> tip_height = locked_chain.getHeight(); - const int max_height = tip_height && *tip_height > 144 ? *tip_height - 144 : 0; // the tip can be reorganized; use a 144-block safety margin - std::map<CKeyID, int> mapKeyFirstBlock; + std::map<CKeyID, const CWalletTx::Confirmation*> mapKeyFirstBlock; + CWalletTx::Confirmation max_confirm; + max_confirm.block_height = GetLastBlockHeight() > 144 ? GetLastBlockHeight() - 144 : 0; // the tip can be reorganized; use a 144-block safety margin + CHECK_NONFATAL(chain().findAncestorByHeight(GetLastBlockHash(), max_confirm.block_height, FoundBlock().hash(max_confirm.hashBlock))); for (const CKeyID &keyid : spk_man->GetKeys()) { if (mapKeyBirth.count(keyid) == 0) - mapKeyFirstBlock[keyid] = max_height; + mapKeyFirstBlock[keyid] = &max_confirm; } // if there are no such keys, we're done @@ -3592,23 +3543,27 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C for (const auto& entry : mapWallet) { // iterate over all wallet transactions... const CWalletTx &wtx = entry.second; - if (Optional<int> height = locked_chain.getBlockHeight(wtx.m_confirm.hashBlock)) { + if (wtx.m_confirm.status == CWalletTx::CONFIRMED) { // ... which are already in a block for (const CTxOut &txout : wtx.tx->vout) { // iterate over all their outputs for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *spk_man)) { // ... and all their affected keys - std::map<CKeyID, int>::iterator rit = mapKeyFirstBlock.find(keyid); - if (rit != mapKeyFirstBlock.end() && *height < rit->second) - rit->second = *height; + auto rit = mapKeyFirstBlock.find(keyid); + if (rit != mapKeyFirstBlock.end() && wtx.m_confirm.block_height < rit->second->block_height) { + rit->second = &wtx.m_confirm; + } } } } } // Extract block timestamps for those keys - for (const auto& entry : mapKeyFirstBlock) - mapKeyBirth[entry.first] = locked_chain.getBlockTime(entry.second) - TIMESTAMP_WINDOW; // block times can be 2h off + for (const auto& entry : mapKeyFirstBlock) { + int64_t block_time; + CHECK_NONFATAL(chain().findBlock(entry.second->hashBlock, FoundBlock().time(block_time))); + mapKeyBirth[entry.first] = block_time - TIMESTAMP_WINDOW; // block times can be 2h off + } } /** @@ -3637,7 +3592,7 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const unsigned int nTimeSmart = wtx.nTimeReceived; if (!wtx.isUnconfirmed() && !wtx.isAbandoned()) { int64_t blocktime; - if (chain().findBlock(wtx.m_confirm.hashBlock, nullptr /* block */, &blocktime)) { + if (chain().findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(blocktime))) { int64_t latestNow = wtx.nTimeReceived; int64_t latestEntry = 0; @@ -3830,44 +3785,6 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - int prev_version = walletInstance->GetVersion(); - if (gArgs.GetBoolArg("-upgradewallet", fFirstRun)) - { - int nMaxVersion = gArgs.GetArg("-upgradewallet", 0); - if (nMaxVersion == 0) // the -upgradewallet without argument case - { - walletInstance->WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); - nMaxVersion = FEATURE_LATEST; - walletInstance->SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately - } - else - walletInstance->WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); - if (nMaxVersion < walletInstance->GetVersion()) - { - error = _("Cannot downgrade wallet").translated; - return nullptr; - } - walletInstance->SetMaxVersion(nMaxVersion); - } - - // Upgrade to HD if explicit upgrade - if (gArgs.GetBoolArg("-upgradewallet", false)) { - LOCK(walletInstance->cs_wallet); - - // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT - int max_version = walletInstance->GetVersion(); - if (!walletInstance->CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { - error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use -upgradewallet=169900 or -upgradewallet with no version specified.").translated; - return nullptr; - } - - for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { - if (!spk_man->Upgrade(prev_version, error)) { - return nullptr; - } - } - } - if (fFirstRun) { // ensure this wallet.dat can only be opened by clients supporting HD with chain split and expects no default key @@ -3875,15 +3792,23 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->SetWalletFlags(wallet_creation_flags, false); - // Always create LegacyScriptPubKeyMan for now - walletInstance->SetupLegacyScriptPubKeyMan(); + // Only create LegacyScriptPubKeyMan when not descriptor wallet + if (!walletInstance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + walletInstance->SetupLegacyScriptPubKeyMan(); + } if (!(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { LOCK(walletInstance->cs_wallet); - for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { - if (!spk_man->SetupGeneration()) { - error = _("Unable to generate initial keys").translated; - return nullptr; + if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + walletInstance->SetupDescriptorScriptPubKeyMans(); + // SetupDescriptorScriptPubKeyMans already calls SetupGeneration for us so we don't need to call SetupGeneration separately + } else { + // Legacy wallets need SetupGeneration here. + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { + if (!spk_man->SetupGeneration()) { + error = _("Unable to generate initial keys").translated; + return nullptr; + } } } } @@ -4064,8 +3989,8 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } { - WalletRescanReserver reserver(walletInstance.get()); - if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), {} /* stop block */, reserver, true /* update */).status)) { + WalletRescanReserver reserver(*walletInstance); + if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { error = _("Failed to rescan the wallet during initialization").translated; return nullptr; } @@ -4129,6 +4054,42 @@ const CAddressBookData* CWallet::FindAddressBookEntry(const CTxDestination& dest return &address_book_it->second; } +bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings) +{ + int prev_version = GetVersion(); + int nMaxVersion = version; + if (nMaxVersion == 0) // the -upgradewallet without argument case + { + WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); + nMaxVersion = FEATURE_LATEST; + SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately + } + else + WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); + if (nMaxVersion < GetVersion()) + { + error = _("Cannot downgrade wallet").translated; + return false; + } + SetMaxVersion(nMaxVersion); + + LOCK(cs_wallet); + + // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT + int max_version = GetVersion(); + if (!CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { + error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified.").translated; + return false; + } + + for (auto spk_man : GetActiveScriptPubKeyMans()) { + if (!spk_man->Upgrade(prev_version, error)) { + return false; + } + } + return true; +} + void CWallet::postInitProcess() { auto locked_chain = chain().lock(); @@ -4186,32 +4147,49 @@ bool CWalletTx::IsImmatureCoinBase() const return GetBlocksToMaturity() > 0; } -std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const { +std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const { std::vector<OutputGroup> groups; std::map<CTxDestination, OutputGroup> gmap; - CTxDestination dst; + std::set<CTxDestination> full_groups; + for (const auto& output : outputs) { if (output.fSpendable) { + CTxDestination dst; CInputCoin input_coin = output.GetInputCoin(); size_t ancestors, descendants; chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); if (!single_coin && ExtractDestination(output.tx->tx->vout[output.i].scriptPubKey, dst)) { - // Limit output groups to no more than 10 entries, to protect - // against inadvertently creating a too-large transaction - // when using -avoidpartialspends - if (gmap[dst].m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { - groups.push_back(gmap[dst]); - gmap.erase(dst); + auto it = gmap.find(dst); + if (it != gmap.end()) { + // Limit output groups to no more than OUTPUT_GROUP_MAX_ENTRIES + // number of entries, to protect against inadvertently creating + // a too-large transaction when using -avoidpartialspends to + // prevent breaking consensus or surprising users with a very + // high amount of fees. + if (it->second.m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { + groups.push_back(it->second); + it->second = OutputGroup{}; + full_groups.insert(dst); + } + it->second.Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); + } else { + gmap[dst].Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); } - gmap[dst].Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); } else { groups.emplace_back(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); } } } if (!single_coin) { - for (const auto& it : gmap) groups.push_back(it.second); + for (auto& it : gmap) { + auto& group = it.second; + if (full_groups.count(it.first) > 0) { + // Make this unattractive as we want coin selection to avoid it if possible + group.m_ancestors = max_ancestors - 1; + } + groups.push_back(group); + } } return groups; } @@ -4341,6 +4319,9 @@ std::unique_ptr<SigningProvider> CWallet::GetSolvingProvider(const CScript& scri LegacyScriptPubKeyMan* CWallet::GetLegacyScriptPubKeyMan() const { + if (IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + return nullptr; + } // Legacy wallets only have one ScriptPubKeyMan which is a LegacyScriptPubKeyMan. // Everything in m_internal_spk_managers and m_external_spk_managers point to the same legacyScriptPubKeyMan. auto it = m_internal_spk_managers.find(OutputType::LEGACY); @@ -4356,7 +4337,7 @@ LegacyScriptPubKeyMan* CWallet::GetOrCreateLegacyScriptPubKeyMan() void CWallet::SetupLegacyScriptPubKeyMan() { - if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty()) { + if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty() || IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { return; } @@ -4385,3 +4366,153 @@ void CWallet::ConnectScriptPubKeyManNotifiers() spk_man->NotifyCanGetAddressesChanged.connect(NotifyCanGetAddressesChanged); } } + +void CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc) +{ + auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc)); + m_spk_managers[id] = std::move(spk_manager); +} + +void CWallet::SetupDescriptorScriptPubKeyMans() +{ + AssertLockHeld(cs_wallet); + + // Make a seed + CKey seed_key; + seed_key.MakeNewKey(true); + CPubKey seed = seed_key.GetPubKey(); + assert(seed_key.VerifyPubKey(seed)); + + // Get the extended key + CExtKey master_key; + master_key.SetSeed(seed_key.begin(), seed_key.size()); + + for (bool internal : {false, true}) { + for (OutputType t : OUTPUT_TYPES) { + auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, t, internal)); + if (IsCrypted()) { + if (IsLocked()) { + throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); + } + if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, nullptr)) { + throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); + } + } + spk_manager->SetupDescriptorGeneration(master_key); + uint256 id = spk_manager->GetID(); + m_spk_managers[id] = std::move(spk_manager); + SetActiveScriptPubKeyMan(id, t, internal); + } + } +} + +void CWallet::SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal, bool memonly) +{ + WalletLogPrintf("Setting spkMan to active: id = %s, type = %d, internal = %d\n", id.ToString(), static_cast<int>(type), static_cast<int>(internal)); + auto& spk_mans = internal ? m_internal_spk_managers : m_external_spk_managers; + auto spk_man = m_spk_managers.at(id).get(); + spk_man->SetType(type, internal); + spk_mans[type] = spk_man; + + if (!memonly) { + WalletBatch batch(*database); + if (!batch.WriteActiveScriptPubKeyMan(static_cast<uint8_t>(type), id, internal)) { + throw std::runtime_error(std::string(__func__) + ": writing active ScriptPubKeyMan id failed"); + } + } + NotifyCanGetAddressesChanged(); +} + +bool CWallet::IsLegacy() const +{ + if (m_internal_spk_managers.count(OutputType::LEGACY) == 0) { + return false; + } + auto spk_man = dynamic_cast<LegacyScriptPubKeyMan*>(m_internal_spk_managers.at(OutputType::LEGACY)); + return spk_man != nullptr; +} + +DescriptorScriptPubKeyMan* CWallet::GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const +{ + for (auto& spk_man_pair : m_spk_managers) { + // Try to downcast to DescriptorScriptPubKeyMan then check if the descriptors match + DescriptorScriptPubKeyMan* spk_manager = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man_pair.second.get()); + if (spk_manager != nullptr && spk_manager->HasWalletDescriptor(desc)) { + return spk_manager; + } + } + + return nullptr; +} + +ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label) +{ + if (!IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + WalletLogPrintf("Cannot add WalletDescriptor to a non-descriptor wallet\n"); + return nullptr; + } + + LOCK(cs_wallet); + auto new_spk_man = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc)); + + // If we already have this descriptor, remove it from the maps but add the existing cache to desc + auto old_spk_man = GetDescriptorScriptPubKeyMan(desc); + if (old_spk_man) { + WalletLogPrintf("Update existing descriptor: %s\n", desc.descriptor->ToString()); + + { + LOCK(old_spk_man->cs_desc_man); + new_spk_man->SetCache(old_spk_man->GetWalletDescriptor().cache); + } + + // Remove from maps of active spkMans + auto old_spk_man_id = old_spk_man->GetID(); + for (bool internal : {false, true}) { + for (OutputType t : OUTPUT_TYPES) { + auto active_spk_man = GetScriptPubKeyMan(t, internal); + if (active_spk_man && active_spk_man->GetID() == old_spk_man_id) { + if (internal) { + m_internal_spk_managers.erase(t); + } else { + m_external_spk_managers.erase(t); + } + break; + } + } + } + m_spk_managers.erase(old_spk_man_id); + } + + // Add the private keys to the descriptor + for (const auto& entry : signing_provider.keys) { + const CKey& key = entry.second; + new_spk_man->AddDescriptorKey(key, key.GetPubKey()); + } + + // Top up key pool, the manager will generate new scriptPubKeys internally + new_spk_man->TopUp(); + + // Apply the label if necessary + // Note: we disable labels for ranged descriptors + if (!desc.descriptor->IsRange()) { + auto script_pub_keys = new_spk_man->GetScriptPubKeys(); + if (script_pub_keys.empty()) { + WalletLogPrintf("Could not generate scriptPubKeys (cache is empty)\n"); + return nullptr; + } + + CTxDestination dest; + if (ExtractDestination(script_pub_keys.at(0), dest)) { + SetAddressBook(dest, label, "receive"); + } + } + + // Save the descriptor to memory + auto ret = new_spk_man.get(); + m_spk_managers[new_spk_man->GetID()] = std::move(new_spk_man); + + // Save the descriptor to DB + ret->WriteDescriptor(); + + return ret; +} diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 6c54c72e76..24c78fceb3 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -111,7 +111,8 @@ static constexpr uint64_t KNOWN_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE | WALLET_FLAG_BLANK_WALLET | WALLET_FLAG_KEY_ORIGIN_METADATA - | WALLET_FLAG_DISABLE_PRIVATE_KEYS; + | WALLET_FLAG_DISABLE_PRIVATE_KEYS + | WALLET_FLAG_DESCRIPTORS; static constexpr uint64_t MUTABLE_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE; @@ -121,6 +122,7 @@ static const std::map<std::string,WalletFlags> WALLET_FLAG_MAP{ {"blank", WALLET_FLAG_BLANK_WALLET}, {"key_origin_metadata", WALLET_FLAG_KEY_ORIGIN_METADATA}, {"disable_private_keys", WALLET_FLAG_DISABLE_PRIVATE_KEYS}, + {"descriptor_wallet", WALLET_FLAG_DESCRIPTORS}, }; extern const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS; @@ -830,7 +832,7 @@ public: bool IsSpentKey(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetSpentKeyState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const; + std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const; bool IsLockedCoin(uint256 hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void LockCoin(const COutPoint& output) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -907,7 +909,7 @@ public: //! USER_ABORT. uint256 last_failed_block; }; - ScanResult ScanForWalletTransactions(const uint256& first_block, const uint256& last_block, const WalletRescanReserver& reserver, bool fUpdate); + ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, Optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate); void transactionRemovedFromMempool(const CTransactionRef &ptx) override; void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); @@ -1163,6 +1165,9 @@ public: returns false if unknown, non-tolerable flags are present */ bool SetWalletFlags(uint64_t overwriteFlags, bool memOnly); + /** Determine if we are a legacy wallet */ + bool IsLegacy() const; + /** Returns a bracketed wallet name for displaying in logs, will return [default wallet] if the wallet has no name */ const std::string GetDisplayName() const override { std::string wallet_name = GetName().length() == 0 ? "default wallet" : GetName(); @@ -1175,6 +1180,9 @@ public: LogPrintf(("%s " + fmt).c_str(), GetDisplayName(), parameters...); }; + /** Upgrade the wallet */ + bool UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings); + //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; @@ -1213,6 +1221,12 @@ public: assert(m_last_block_processed_height >= 0); return m_last_block_processed_height; }; + uint256 GetLastBlockHash() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) + { + AssertLockHeld(cs_wallet); + assert(m_last_block_processed_height >= 0); + return m_last_block_processed; + } /** Set last block processed height, currently only use in unit test */ void SetLastBlockProcessed(int block_height, uint256 block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { @@ -1223,6 +1237,25 @@ public: //! Connect the signals from ScriptPubKeyMans to the signals in CWallet void ConnectScriptPubKeyManNotifiers(); + + //! Instantiate a descriptor ScriptPubKeyMan from the WalletDescriptor and load it + void LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc); + + //! Sets the active ScriptPubKeyMan for the specified type and internal + //! @param[in] id The unique id for the ScriptPubKeyMan + //! @param[in] type The OutputType this ScriptPubKeyMan provides addresses for + //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses + //! @param[in] memonly Whether to record this update to the database. Set to true for wallet loading, normally false when actually updating the wallet. + void SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal, bool memonly = false); + + //! Create new DescriptorScriptPubKeyMans and add them to the wallet + void SetupDescriptorScriptPubKeyMans(); + + //! Return the DescriptorScriptPubKeyMan for a WalletDescriptor if it is already in the wallet + DescriptorScriptPubKeyMan* GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const; + + //! Add a descriptor to the wallet, return a ScriptPubKeyMan & associated output type + ScriptPubKeyMan* AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label); }; /** @@ -1235,35 +1268,35 @@ void MaybeResendWalletTxs(); class WalletRescanReserver { private: - CWallet* m_wallet; + CWallet& m_wallet; bool m_could_reserve; public: - explicit WalletRescanReserver(CWallet* w) : m_wallet(w), m_could_reserve(false) {} + explicit WalletRescanReserver(CWallet& w) : m_wallet(w), m_could_reserve(false) {} bool reserve() { assert(!m_could_reserve); - std::lock_guard<std::mutex> lock(m_wallet->mutexScanning); - if (m_wallet->fScanningWallet) { + std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); + if (m_wallet.fScanningWallet) { return false; } - m_wallet->m_scanning_start = GetTimeMillis(); - m_wallet->m_scanning_progress = 0; - m_wallet->fScanningWallet = true; + m_wallet.m_scanning_start = GetTimeMillis(); + m_wallet.m_scanning_progress = 0; + m_wallet.fScanningWallet = true; m_could_reserve = true; return true; } bool isReserved() const { - return (m_could_reserve && m_wallet->fScanningWallet); + return (m_could_reserve && m_wallet.fScanningWallet); } ~WalletRescanReserver() { - std::lock_guard<std::mutex> lock(m_wallet->mutexScanning); + std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); if (m_could_reserve) { - m_wallet->fScanningWallet = false; + m_wallet.fScanningWallet = false; } } }; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 568b21ed00..79316ca3e7 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,6 +21,8 @@ namespace DBKeys { const std::string ACENTRY{"acentry"}; +const std::string ACTIVEEXTERNALSPK{"activeexternalspk"}; +const std::string ACTIVEINTERNALSPK{"activeinternalspk"}; const std::string BESTBLOCK_NOMERKLE{"bestblock_nomerkle"}; const std::string BESTBLOCK{"bestblock"}; const std::string CRYPTED_KEY{"ckey"}; @@ -41,6 +43,10 @@ const std::string PURPOSE{"purpose"}; const std::string SETTINGS{"settings"}; const std::string TX{"tx"}; const std::string VERSION{"version"}; +const std::string WALLETDESCRIPTOR{"walletdescriptor"}; +const std::string WALLETDESCRIPTORCACHE{"walletdescriptorcache"}; +const std::string WALLETDESCRIPTORCKEY{"walletdescriptorckey"}; +const std::string WALLETDESCRIPTORKEY{"walletdescriptorkey"}; const std::string WATCHMETA{"watchmeta"}; const std::string WATCHS{"watchs"}; } // namespace DBKeys @@ -179,6 +185,51 @@ bool WalletBatch::WriteMinVersion(int nVersion) return WriteIC(DBKeys::MINVERSION, nVersion); } +bool WalletBatch::WriteActiveScriptPubKeyMan(uint8_t type, const uint256& id, bool internal) +{ + std::string key = internal ? DBKeys::ACTIVEINTERNALSPK : DBKeys::ACTIVEEXTERNALSPK; + return WriteIC(make_pair(key, type), id); +} + +bool WalletBatch::WriteDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const CPrivKey& privkey) +{ + // hash pubkey/privkey to accelerate wallet load + std::vector<unsigned char> key; + key.reserve(pubkey.size() + privkey.size()); + key.insert(key.end(), pubkey.begin(), pubkey.end()); + key.insert(key.end(), privkey.begin(), privkey.end()); + + return WriteIC(std::make_pair(DBKeys::WALLETDESCRIPTORKEY, std::make_pair(desc_id, pubkey)), std::make_pair(privkey, Hash(key.begin(), key.end())), false); +} + +bool WalletBatch::WriteCryptedDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const std::vector<unsigned char>& secret) +{ + if (!WriteIC(std::make_pair(DBKeys::WALLETDESCRIPTORCKEY, std::make_pair(desc_id, pubkey)), secret, false)) { + return false; + } + EraseIC(std::make_pair(DBKeys::WALLETDESCRIPTORKEY, std::make_pair(desc_id, pubkey))); + return true; +} + +bool WalletBatch::WriteDescriptor(const uint256& desc_id, const WalletDescriptor& descriptor) +{ + return WriteIC(make_pair(DBKeys::WALLETDESCRIPTOR, desc_id), descriptor); +} + +bool WalletBatch::WriteDescriptorDerivedCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index, uint32_t der_index) +{ + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + xpub.Encode(ser_xpub.data()); + return WriteIC(std::make_pair(std::make_pair(DBKeys::WALLETDESCRIPTORCACHE, desc_id), std::make_pair(key_exp_index, der_index)), ser_xpub); +} + +bool WalletBatch::WriteDescriptorParentCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index) +{ + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + xpub.Encode(ser_xpub.data()); + return WriteIC(std::make_pair(std::make_pair(DBKeys::WALLETDESCRIPTORCACHE, desc_id), key_exp_index), ser_xpub); +} + class CWalletScanState { public: unsigned int nKeys{0}; @@ -189,6 +240,11 @@ public: bool fIsEncrypted{false}; bool fAnyUnordered{false}; std::vector<uint256> vWalletUpgrade; + std::map<OutputType, uint256> m_active_external_spks; + std::map<OutputType, uint256> m_active_internal_spks; + std::map<uint256, DescriptorCache> m_descriptor_caches; + std::map<std::pair<uint256, CKeyID>, CKey> m_descriptor_keys; + std::map<std::pair<uint256, CKeyID>, std::pair<CPubKey, std::vector<unsigned char>>> m_descriptor_crypt_keys; CWalletScanState() { } @@ -404,6 +460,108 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == DBKeys::OLD_KEY) { strErr = "Found unsupported 'wkey' record, try loading with version 0.18"; return false; + } else if (strType == DBKeys::ACTIVEEXTERNALSPK || strType == DBKeys::ACTIVEINTERNALSPK) { + uint8_t type; + ssKey >> type; + uint256 id; + ssValue >> id; + + bool internal = strType == DBKeys::ACTIVEINTERNALSPK; + auto& spk_mans = internal ? wss.m_active_internal_spks : wss.m_active_external_spks; + if (spk_mans.count(static_cast<OutputType>(type)) > 0) { + strErr = "Multiple ScriptPubKeyMans specified for a single type"; + return false; + } + spk_mans[static_cast<OutputType>(type)] = id; + } else if (strType == DBKeys::WALLETDESCRIPTOR) { + uint256 id; + ssKey >> id; + WalletDescriptor desc; + ssValue >> desc; + if (wss.m_descriptor_caches.count(id) == 0) { + wss.m_descriptor_caches[id] = DescriptorCache(); + } + pwallet->LoadDescriptorScriptPubKeyMan(id, desc); + } else if (strType == DBKeys::WALLETDESCRIPTORCACHE) { + bool parent = true; + uint256 desc_id; + uint32_t key_exp_index; + uint32_t der_index; + ssKey >> desc_id; + ssKey >> key_exp_index; + + // if the der_index exists, it's a derived xpub + try + { + ssKey >> der_index; + parent = false; + } + catch (...) {} + + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + ssValue >> ser_xpub; + CExtPubKey xpub; + xpub.Decode(ser_xpub.data()); + if (wss.m_descriptor_caches.count(desc_id)) { + wss.m_descriptor_caches[desc_id] = DescriptorCache(); + } + if (parent) { + wss.m_descriptor_caches[desc_id].CacheParentExtPubKey(key_exp_index, xpub); + } else { + wss.m_descriptor_caches[desc_id].CacheDerivedExtPubKey(key_exp_index, der_index, xpub); + } + } else if (strType == DBKeys::WALLETDESCRIPTORKEY) { + uint256 desc_id; + CPubKey pubkey; + ssKey >> desc_id; + ssKey >> pubkey; + if (!pubkey.IsValid()) + { + strErr = "Error reading wallet database: CPubKey corrupt"; + return false; + } + CKey key; + CPrivKey pkey; + uint256 hash; + + wss.nKeys++; + ssValue >> pkey; + ssValue >> hash; + + // hash pubkey/privkey to accelerate wallet load + std::vector<unsigned char> to_hash; + to_hash.reserve(pubkey.size() + pkey.size()); + to_hash.insert(to_hash.end(), pubkey.begin(), pubkey.end()); + to_hash.insert(to_hash.end(), pkey.begin(), pkey.end()); + + if (Hash(to_hash.begin(), to_hash.end()) != hash) + { + strErr = "Error reading wallet database: CPubKey/CPrivKey corrupt"; + return false; + } + + if (!key.Load(pkey, pubkey, true)) + { + strErr = "Error reading wallet database: CPrivKey corrupt"; + return false; + } + wss.m_descriptor_keys.insert(std::make_pair(std::make_pair(desc_id, pubkey.GetID()), key)); + } else if (strType == DBKeys::WALLETDESCRIPTORCKEY) { + uint256 desc_id; + CPubKey pubkey; + ssKey >> desc_id; + ssKey >> pubkey; + if (!pubkey.IsValid()) + { + strErr = "Error reading wallet database: CPubKey corrupt"; + return false; + } + std::vector<unsigned char> privkey; + ssValue >> privkey; + wss.nCKeys++; + + wss.m_descriptor_crypt_keys.insert(std::make_pair(std::make_pair(desc_id, pubkey.GetID()), std::make_pair(pubkey, privkey))); + wss.fIsEncrypted = true; } else if (strType != DBKeys::BESTBLOCK && strType != DBKeys::BESTBLOCK_NOMERKLE && strType != DBKeys::MINVERSION && strType != DBKeys::ACENTRY && strType != DBKeys::VERSION && strType != DBKeys::SETTINGS) { @@ -497,6 +655,31 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) result = DBErrors::CORRUPT; } + // Set the active ScriptPubKeyMans + for (auto spk_man_pair : wss.m_active_external_spks) { + pwallet->SetActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ false, /* memonly */ true); + } + for (auto spk_man_pair : wss.m_active_internal_spks) { + pwallet->SetActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ true, /* memonly */ true); + } + + // Set the descriptor caches + for (auto desc_cache_pair : wss.m_descriptor_caches) { + auto spk_man = pwallet->GetScriptPubKeyMan(desc_cache_pair.first); + assert(spk_man); + ((DescriptorScriptPubKeyMan*)spk_man)->SetCache(desc_cache_pair.second); + } + + // Set the descriptor keys + for (auto desc_key_pair : wss.m_descriptor_keys) { + auto spk_man = pwallet->GetScriptPubKeyMan(desc_key_pair.first.first); + ((DescriptorScriptPubKeyMan*)spk_man)->AddKey(desc_key_pair.first.second, desc_key_pair.second); + } + for (auto desc_key_pair : wss.m_descriptor_crypt_keys) { + auto spk_man = pwallet->GetScriptPubKeyMan(desc_key_pair.first.first); + ((DescriptorScriptPubKeyMan*)spk_man)->AddCryptedKey(desc_key_pair.first.second, desc_key_pair.second.first, desc_key_pair.second.second); + } + if (fNoncriticalErrors && result == DBErrors::LOAD_OK) result = DBErrors::NONCRITICAL_ERROR; @@ -516,7 +699,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys, wss.m_unknown_records); // nTimeFirstKey is only reliable if all keys have metadata - if ((wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { + if (pwallet->IsLegacy() && (wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { auto spk_man = pwallet->GetOrCreateLegacyScriptPubKeyMan(); if (spk_man) { LOCK(spk_man->cs_KeyStore); diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 1a65125480..2701481c58 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,6 +9,7 @@ #include <amount.h> #include <script/sign.h> #include <wallet/db.h> +#include <wallet/walletutil.h> #include <key.h> #include <stdint.h> @@ -54,6 +55,8 @@ enum class DBErrors namespace DBKeys { extern const std::string ACENTRY; +extern const std::string ACTIVEEXTERNALSPK; +extern const std::string ACTIVEINTERNALSPK; extern const std::string BESTBLOCK; extern const std::string BESTBLOCK_NOMERKLE; extern const std::string CRYPTED_KEY; @@ -74,6 +77,9 @@ extern const std::string PURPOSE; extern const std::string SETTINGS; extern const std::string TX; extern const std::string VERSION; +extern const std::string WALLETDESCRIPTOR; +extern const std::string WALLETDESCRIPTORCKEY; +extern const std::string WALLETDESCRIPTORKEY; extern const std::string WATCHMETA; extern const std::string WATCHS; } // namespace DBKeys @@ -240,11 +246,19 @@ public: bool WriteMinVersion(int nVersion); + bool WriteDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const CPrivKey& privkey); + bool WriteCryptedDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const std::vector<unsigned char>& secret); + bool WriteDescriptor(const uint256& desc_id, const WalletDescriptor& descriptor); + bool WriteDescriptorDerivedCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index, uint32_t der_index); + bool WriteDescriptorParentCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index); + /// Write destination data key,value tuple to database bool WriteDestData(const std::string &address, const std::string &key, const std::string &value); /// Erase destination data tuple from wallet database bool EraseDestData(const std::string &address, const std::string &key); + bool WriteActiveScriptPubKeyMan(uint8_t type, const uint256& id, bool internal); + DBErrors LoadWallet(CWallet* pwallet); DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx); DBErrors ZapWalletTx(std::vector<CWalletTx>& vWtx); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 8c918f4eb0..65643669c2 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 04c2407a89..8bac0608a9 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -100,5 +100,10 @@ WalletLocation::WalletLocation(const std::string& name) bool WalletLocation::Exists() const { - return fs::symlink_status(m_path).type() != fs::file_not_found; + fs::path path = m_path; + // For the default wallet, check specifically for the wallet.dat file + if (m_name.empty()) { + path = fs::absolute("wallet.dat", m_path); + } + return fs::symlink_status(path).type() != fs::file_not_found; } diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index c91c9aca96..d7e07ed04c 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -6,6 +6,7 @@ #define BITCOIN_WALLET_WALLETUTIL_H #include <fs.h> +#include <script/descriptor.h> #include <vector> @@ -55,6 +56,9 @@ enum WalletFlags : uint64_t { //! bitcoin from opening the wallet, thinking it was newly created, and //! then improperly reinitializing it. WALLET_FLAG_BLANK_WALLET = (1ULL << 33), + + //! Indicate that this wallet supports DescriptorScriptPubKeyMan + WALLET_FLAG_DESCRIPTORS = (1ULL << 34), }; //! Get the path of the wallet directory. @@ -83,4 +87,41 @@ public: bool Exists() const; }; +/** Descriptor with some wallet metadata */ +class WalletDescriptor +{ +public: + std::shared_ptr<Descriptor> descriptor; + uint64_t creation_time; + int32_t range_start; // First item in range; start of range, inclusive, i.e. [range_start, range_end). This never changes. + int32_t range_end; // Item after the last; end of range, exclusive, i.e. [range_start, range_end). This will increment with each TopUp() + int32_t next_index; // Position of the next item to generate + DescriptorCache cache; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + if (ser_action.ForRead()) { + std::string desc; + std::string error; + READWRITE(desc); + FlatSigningProvider keys; + descriptor = Parse(desc, keys, error, true); + if (!descriptor) { + throw std::ios_base::failure("Invalid descriptor: " + error); + } + } else { + READWRITE(descriptor->ToString()); + } + READWRITE(creation_time); + READWRITE(next_index); + READWRITE(range_start); + READWRITE(range_end); + } + + WalletDescriptor() {} + WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) {} +}; + #endif // BITCOIN_WALLET_WALLETUTIL_H diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index 0fbefb6023..cce6210129 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index ce14998fd1..ae5721bec2 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index e47e709431..75267de80b 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind aborts if can't disconnect a block. diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index ef4d9411c5..79777f5582 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for skipping signature validation on old blocks. diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index adf66243b7..166c28d376 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Backwards compatibility functional test @@ -94,7 +94,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # w1: regular wallet, created on master: update this test when default # wallets can no longer be opened by older versions. - node_master.createwallet(wallet_name="w1") + node_master.rpc.createwallet(wallet_name="w1") wallet = node_master.get_wallet_rpc("w1") info = wallet.getwalletinfo() assert info['private_keys_enabled'] @@ -120,17 +120,17 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.nodes[1].abandontransaction(tx3_id) # w1_v19: regular wallet, created with v0.19 - node_v19.createwallet(wallet_name="w1_v19") + node_v19.rpc.createwallet(wallet_name="w1_v19") wallet = node_v19.get_wallet_rpc("w1_v19") info = wallet.getwalletinfo() assert info['private_keys_enabled'] assert info['keypoolsize'] > 0 # Use addmultisigaddress (see #18075) - address_18075 = wallet.addmultisigaddress(1, ["0296b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52", "037211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073"], "", "legacy")["address"] + address_18075 = wallet.rpc.addmultisigaddress(1, ["0296b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52", "037211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073"], "", "legacy")["address"] assert wallet.getaddressinfo(address_18075)["solvable"] # w1_v18: regular wallet, created with v0.18 - node_v18.createwallet(wallet_name="w1_v18") + node_v18.rpc.createwallet(wallet_name="w1_v18") wallet = node_v18.get_wallet_rpc("w1_v18") info = wallet.getwalletinfo() assert info['private_keys_enabled'] @@ -139,21 +139,21 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # w2: wallet with private keys disabled, created on master: update this # test when default wallets private keys disabled can no longer be # opened by older versions. - node_master.createwallet(wallet_name="w2", disable_private_keys=True) + node_master.rpc.createwallet(wallet_name="w2", disable_private_keys=True) wallet = node_master.get_wallet_rpc("w2") info = wallet.getwalletinfo() assert info['private_keys_enabled'] == False assert info['keypoolsize'] == 0 # w2_v19: wallet with private keys disabled, created with v0.19 - node_v19.createwallet(wallet_name="w2_v19", disable_private_keys=True) + node_v19.rpc.createwallet(wallet_name="w2_v19", disable_private_keys=True) wallet = node_v19.get_wallet_rpc("w2_v19") info = wallet.getwalletinfo() assert info['private_keys_enabled'] == False assert info['keypoolsize'] == 0 # w2_v18: wallet with private keys disabled, created with v0.18 - node_v18.createwallet(wallet_name="w2_v18", disable_private_keys=True) + node_v18.rpc.createwallet(wallet_name="w2_v18", disable_private_keys=True) wallet = node_v18.get_wallet_rpc("w2_v18") info = wallet.getwalletinfo() assert info['private_keys_enabled'] == False @@ -161,21 +161,21 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # w3: blank wallet, created on master: update this # test when default blank wallets can no longer be opened by older versions. - node_master.createwallet(wallet_name="w3", blank=True) + node_master.rpc.createwallet(wallet_name="w3", blank=True) wallet = node_master.get_wallet_rpc("w3") info = wallet.getwalletinfo() assert info['private_keys_enabled'] assert info['keypoolsize'] == 0 # w3_v19: blank wallet, created with v0.19 - node_v19.createwallet(wallet_name="w3_v19", blank=True) + node_v19.rpc.createwallet(wallet_name="w3_v19", blank=True) wallet = node_v19.get_wallet_rpc("w3_v19") info = wallet.getwalletinfo() assert info['private_keys_enabled'] assert info['keypoolsize'] == 0 # w3_v18: blank wallet, created with v0.18 - node_v18.createwallet(wallet_name="w3_v18", blank=True) + node_v18.rpc.createwallet(wallet_name="w3_v18", blank=True) wallet = node_v18.get_wallet_rpc("w3_v18") info = wallet.getwalletinfo() assert info['private_keys_enabled'] @@ -318,7 +318,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.log.info("Test wallet upgrade path...") # u1: regular wallet, created with v0.17 - node_v17.createwallet(wallet_name="u1_v17") + node_v17.rpc.createwallet(wallet_name="u1_v17") wallet = node_v17.get_wallet_rpc("u1_v17") address = wallet.getnewaddress("bech32") info = wallet.getaddressinfo(address) diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index fe45ed34b5..549e8b2029 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -324,7 +324,7 @@ class BIP68Test(BitcoinTestFramework): block.solve() tip = block.sha256 height += 1 - self.nodes[0].submitblock(ToHex(block)) + assert_equal(None if i == 1 else 'inconclusive', self.nodes[0].submitblock(ToHex(block))) cur_time += 1 mempool = self.nodes[0].getrawmempool() @@ -381,7 +381,7 @@ class BIP68Test(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(None, self.nodes[0].submitblock(block.serialize().hex())) assert_equal(self.nodes[0].getbestblockhash(), block.hash) def activateCSV(self): diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 0c591de869..6619d83dc4 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test block processing.""" diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 073ed8d7c7..fd0330924d 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP65 (CHECKLOCKTIMEVERIFY). diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index a98480a6dd..c6852ef017 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test CSV soft fork activation. diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 38cdf0501e..05fdacd451 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP66 (DER SIG). diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 5128485ec0..3cf0fb8f7b 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index be1c044aa5..b56ffe179f 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check that it's not possible to start a second bitcoind instance using the same datadir or wallet.""" diff --git a/test/functional/feature_help.py b/test/functional/feature_help.py index e3e2456183..9f18413255 100755 --- a/test/functional/feature_help.py +++ b/test/functional/feature_help.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Verify that starting bitcoin with -h works as expected.""" diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index 7f1e8f5bae..1138b0f0ea 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test loadblock option diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py index 1bae29b302..e4bf2d849d 100755 --- a/test/functional/feature_logging.py +++ b/test/functional/feature_logging.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test debug logging.""" diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index dc00e03fe7..d4a8f8a715 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of -maxuploadtarget. diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index 9c92ee7f90..ff55cb76d9 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test NULLDUMMY softfork. @@ -111,7 +111,7 @@ class NULLDUMMYTest(BitcoinTestFramework): witness and add_witness_commitment(block) block.rehash() block.solve() - node.submitblock(block.serialize().hex()) + assert_equal(None if accept else 'block-validation-failed', node.submitblock(block.serialize().hex())) if (accept): assert_equal(node.getbestblockhash(), block.hash) self.tip = block.sha256 diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 21b6b299f6..c9362cf5aa 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the pruning code. diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 9e578f0026..acf551ef69 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -10,7 +10,7 @@ from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut from test_framework.script import CScript, OP_DROP from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, satoshi_round -from test_framework.script_util import DUMMY_P2WPKH_SCRIPT +from test_framework.script_util import DUMMY_P2WPKH_SCRIPT, DUMMY_2_P2WPKH_SCRIPT MAX_REPLACEMENT_LIMIT = 100 @@ -142,7 +142,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): # Should fail because we haven't changed the fee tx1b = CTransaction() tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT + b'a')] + tx1b.vout = [CTxOut(1 * COIN, DUMMY_2_P2WPKH_SCRIPT)] tx1b_hex = txToHex(tx1b) # This will raise an exception due to insufficient fee diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 909a43c8d9..fdd86310c0 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the SegWit changeover logic.""" @@ -28,8 +28,8 @@ from test_framework.util import ( NODE_0 = 0 NODE_2 = 2 -WIT_V0 = 0 -WIT_V1 = 1 +P2WPKH = 0 +P2WSH = 1 def getutxo(txid): utxo = {} @@ -118,8 +118,8 @@ class SegWitTest(BitcoinTestFramework): balance_presetup = self.nodes[0].getbalance() self.pubkey = [] - p2sh_ids = [] # p2sh_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE embedded in p2sh - wit_ids = [] # wit_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE via bare witness + p2sh_ids = [] # p2sh_ids[NODE][TYPE] is an array of txids that spend to P2WPKH (TYPE=0) or P2WSH (TYPE=1) scripts to an address for NODE embedded in p2sh + wit_ids = [] # wit_ids[NODE][TYPE] is an array of txids that spend to P2WPKH (TYPE=0) or P2WSH (TYPE=1) scripts to an address for NODE via bare witness for i in range(3): newaddress = self.nodes[i].getnewaddress() self.pubkey.append(self.nodes[i].getaddressinfo(newaddress)["pubkey"]) @@ -152,14 +152,14 @@ class SegWitTest(BitcoinTestFramework): self.sync_blocks() self.log.info("Verify witness txs are skipped for mining before the fork") - self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][0], True) # block 424 - self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][0], True) # block 425 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][0], True) # block 426 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][0], True) # block 427 + self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WPKH][0], True) # block 424 + self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WSH][0], True) # block 425 + self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][P2WPKH][0], True) # block 426 + self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][P2WSH][0], True) # block 427 self.log.info("Verify unsigned p2sh witness txs without a redeem script are invalid") - self.fail_accept(self.nodes[2], "mandatory-script-verify-flag", p2sh_ids[NODE_2][WIT_V0][1], False) - self.fail_accept(self.nodes[2], "mandatory-script-verify-flag", p2sh_ids[NODE_2][WIT_V1][1], False) + self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WPKH][1], sign=False) + self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WSH][1], sign=False) self.nodes[2].generate(4) # blocks 428-431 @@ -173,13 +173,13 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify default node can't accept txs with missing witness") # unsigned, no scriptsig - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", wit_ids[NODE_0][WIT_V0][0], False) - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", wit_ids[NODE_0][WIT_V1][0], False) - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V0][0], False) - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False) + self.fail_accept(self.nodes[0], "non-mandatory-script-verify-flag (Witness program hash mismatch)", wit_ids[NODE_0][P2WPKH][0], sign=False) + self.fail_accept(self.nodes[0], "non-mandatory-script-verify-flag (Witness program was passed an empty witness)", wit_ids[NODE_0][P2WSH][0], sign=False) + self.fail_accept(self.nodes[0], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_0][P2WPKH][0], sign=False) + self.fail_accept(self.nodes[0], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_0][P2WSH][0], sign=False) # unsigned with redeem script - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V0][0], False, witness_script(False, self.pubkey[0])) - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False, witness_script(True, self.pubkey[0])) + self.fail_accept(self.nodes[0], "non-mandatory-script-verify-flag (Witness program hash mismatch)", p2sh_ids[NODE_0][P2WPKH][0], sign=False, redeem_script=witness_script(False, self.pubkey[0])) + self.fail_accept(self.nodes[0], "non-mandatory-script-verify-flag (Witness program was passed an empty witness)", p2sh_ids[NODE_0][P2WSH][0], sign=False, redeem_script=witness_script(True, self.pubkey[0])) self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") assert self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False) @@ -194,16 +194,16 @@ class SegWitTest(BitcoinTestFramework): assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) == tx.serialize_without_witness().hex() self.log.info("Verify witness txs without witness data are invalid after the fork") - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', wit_ids[NODE_2][WIT_V0][2], sign=False) - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', wit_ids[NODE_2][WIT_V1][2], sign=False) - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', p2sh_ids[NODE_2][WIT_V0][2], sign=False, redeem_script=witness_script(False, self.pubkey[2])) - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', p2sh_ids[NODE_2][WIT_V1][2], sign=False, redeem_script=witness_script(True, self.pubkey[2])) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', wit_ids[NODE_2][P2WPKH][2], sign=False) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', wit_ids[NODE_2][P2WSH][2], sign=False) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', p2sh_ids[NODE_2][P2WPKH][2], sign=False, redeem_script=witness_script(False, self.pubkey[2])) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', p2sh_ids[NODE_2][P2WSH][2], sign=False, redeem_script=witness_script(True, self.pubkey[2])) self.log.info("Verify default node can now use witness txs") - self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], True) # block 432 - self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V1][0], True) # block 433 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], True) # block 434 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], True) # block 435 + self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WPKH][0], True) # block 432 + self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WSH][0], True) # block 433 + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WPKH][0], True) # block 434 + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WSH][0], True) # block 435 self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index f04a58cd19..1c94305220 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -1,13 +1,19 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_process_error, get_auth_cookie -class TestBitcoinCli(BitcoinTestFramework): +# The block reward of coinbaseoutput.nValue (50) BTC/block matures after +# COINBASE_MATURITY (100) blocks. Therefore, after mining 101 blocks we expect +# node 0 to have a balance of (BLOCKS - COINBASE_MATURITY) * 50 BTC/block. +BLOCKS = 101 +BALANCE = (BLOCKS - 100) * 50 +class TestBitcoinCli(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 @@ -17,15 +23,7 @@ class TestBitcoinCli(BitcoinTestFramework): def run_test(self): """Main test logic""" - - cli_response = self.nodes[0].cli("-version").send_cli() - assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response - - self.log.info("Compare responses from getwalletinfo RPC and `bitcoin-cli getwalletinfo`") - if self.is_wallet_compiled(): - cli_response = self.nodes[0].cli.getwalletinfo() - rpc_response = self.nodes[0].getwalletinfo() - assert_equal(cli_response, rpc_response) + self.nodes[0].generate(BLOCKS) self.log.info("Compare responses from getblockchaininfo RPC and `bitcoin-cli getblockchaininfo`") cli_response = self.nodes[0].cli.getblockchaininfo() @@ -35,12 +33,12 @@ class TestBitcoinCli(BitcoinTestFramework): user, password = get_auth_cookie(self.nodes[0].datadir, self.chain) self.log.info("Test -stdinrpcpass option") - assert_equal(0, self.nodes[0].cli('-rpcuser=%s' % user, '-stdinrpcpass', input=password).getblockcount()) - assert_raises_process_error(1, "Incorrect rpcuser or rpcpassword", self.nodes[0].cli('-rpcuser=%s' % user, '-stdinrpcpass', input="foo").echo) + assert_equal(BLOCKS, self.nodes[0].cli('-rpcuser={}'.format(user), '-stdinrpcpass', input=password).getblockcount()) + assert_raises_process_error(1, 'Incorrect rpcuser or rpcpassword', self.nodes[0].cli('-rpcuser={}'.format(user), '-stdinrpcpass', input='foo').echo) self.log.info("Test -stdin and -stdinrpcpass") - assert_equal(["foo", "bar"], self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input=password + "\nfoo\nbar").echo()) - assert_raises_process_error(1, "Incorrect rpcuser or rpcpassword", self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input="foo").echo) + assert_equal(['foo', 'bar'], self.nodes[0].cli('-rpcuser={}'.format(user), '-stdin', '-stdinrpcpass', input=password + '\nfoo\nbar').echo()) + assert_raises_process_error(1, 'Incorrect rpcuser or rpcpassword', self.nodes[0].cli('-rpcuser={}'.format(user), '-stdin', '-stdinrpcpass', input='foo').echo) self.log.info("Test connecting to a non-existing server") assert_raises_process_error(1, "Could not connect to the server", self.nodes[0].cli('-rpcport=1').echo) @@ -48,29 +46,87 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test connecting with non-existing RPC cookie file") assert_raises_process_error(1, "Could not locate RPC credentials", self.nodes[0].cli('-rpccookiefile=does-not-exist', '-rpcpassword=').echo) - self.log.info("Make sure that -getinfo with arguments fails") + self.log.info("Test -getinfo with arguments fails") assert_raises_process_error(1, "-getinfo takes no arguments", self.nodes[0].cli('-getinfo').help) - self.log.info("Compare responses from `bitcoin-cli -getinfo` and the RPCs data is retrieved from.") - cli_get_info = self.nodes[0].cli('-getinfo').send_cli() + self.log.info("Test -getinfo returns expected network and blockchain info") if self.is_wallet_compiled(): - wallet_info = self.nodes[0].getwalletinfo() + self.nodes[0].encryptwallet(password) + cli_get_info = self.nodes[0].cli('-getinfo').send_cli() network_info = self.nodes[0].getnetworkinfo() blockchain_info = self.nodes[0].getblockchaininfo() - assert_equal(cli_get_info['version'], network_info['version']) assert_equal(cli_get_info['blocks'], blockchain_info['blocks']) + assert_equal(cli_get_info['headers'], blockchain_info['headers']) assert_equal(cli_get_info['timeoffset'], network_info['timeoffset']) assert_equal(cli_get_info['connections'], network_info['connections']) assert_equal(cli_get_info['proxy'], network_info['networks'][0]['proxy']) assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) assert_equal(cli_get_info['chain'], blockchain_info['chain']) + if self.is_wallet_compiled(): - assert_equal(cli_get_info['balance'], wallet_info['balance']) + self.log.info("Test -getinfo and bitcoin-cli getwalletinfo return expected wallet info") + assert_equal(cli_get_info['balance'], BALANCE) + wallet_info = self.nodes[0].getwalletinfo() assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) + assert_equal(cli_get_info['unlocked_until'], wallet_info['unlocked_until']) assert_equal(cli_get_info['paytxfee'], wallet_info['paytxfee']) assert_equal(cli_get_info['relayfee'], network_info['relayfee']) - # unlocked_until is not tested because the wallet is not encrypted + assert_equal(self.nodes[0].cli.getwalletinfo(), wallet_info) + + # Setup to test -getinfo and -rpcwallet= with multiple wallets. + wallets = ['', 'Encrypted', 'secret'] + amounts = [Decimal('59.999928'), Decimal(9), Decimal(31)] + self.nodes[0].createwallet(wallet_name=wallets[1]) + self.nodes[0].createwallet(wallet_name=wallets[2]) + w1 = self.nodes[0].get_wallet_rpc(wallets[0]) + w2 = self.nodes[0].get_wallet_rpc(wallets[1]) + w3 = self.nodes[0].get_wallet_rpc(wallets[2]) + w1.walletpassphrase(password, self.rpc_timeout) + w1.sendtoaddress(w2.getnewaddress(), amounts[1]) + w1.sendtoaddress(w3.getnewaddress(), amounts[2]) + + # Mine a block to confirm; adds a block reward (50 BTC) to the default wallet. + self.nodes[0].generate(1) + + self.log.info("Test -getinfo with multiple wallets loaded returns no balance") + assert_equal(set(self.nodes[0].listwallets()), set(wallets)) + assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli().keys() + + self.log.info("Test -getinfo with multiple wallets and -rpcwallet returns specified wallet balance") + for i in range(len(wallets)): + cli_get_info = self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[i])) + assert_equal(cli_get_info['balance'], amounts[i]) + + self.log.info("Test -getinfo with multiple wallets and -rpcwallet=non-existing-wallet returns no balance") + assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli('-rpcwallet=does-not-exist').keys() + + self.log.info("Test -getinfo after unloading all wallets except a non-default one returns its balance") + self.nodes[0].unloadwallet(wallets[0]) + self.nodes[0].unloadwallet(wallets[2]) + assert_equal(self.nodes[0].listwallets(), [wallets[1]]) + assert_equal(self.nodes[0].cli('-getinfo').send_cli()['balance'], amounts[1]) + + self.log.info("Test -getinfo -rpcwallet=remaining-non-default-wallet returns its balance") + assert_equal(self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[1]))['balance'], amounts[1]) + + self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balance") + assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[2])).keys() + else: + self.log.info("*** Wallet not compiled; cli getwalletinfo and -getinfo wallet tests skipped") + self.nodes[0].generate(1) # maintain block parity with the wallet_compiled conditional branch + + self.log.info("Test -version with node stopped") + self.stop_node(0) + cli_response = self.nodes[0].cli('-version').send_cli() + assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response + + self.log.info("Test -rpcwait option successfully waits for RPC connection") + self.nodes[0].start() # start node without RPC connection + self.nodes[0].wait_for_cookie_credentials() # ensure cookie file is available to avoid race condition + blocks = self.nodes[0].cli('-rpcwait').send_cli('getblockcount') + self.nodes[0].wait_for_rpc_connection() + assert_equal(blocks, BLOCKS + 1) if __name__ == '__main__': diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index 3b3b5bb0c1..ac2e73fc5c 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests some generic aspects of the RPC interface.""" diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 9ade22a7eb..6df8f1c3ec 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool acceptance of raw transactions.""" diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index d3690b245e..e1671624a8 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool persistence. diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index d797dff134..8e1f87e42c 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool re-org scenarios. @@ -16,12 +16,16 @@ from test_framework.util import assert_equal, assert_raises_rpc_error class MempoolCoinbaseTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + self.extra_args = [ + [ + '-whitelist=noban@127.0.0.1', # immediate tx relay + ], + [] + ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() - alert_filename = None # Set by setup_network - def run_test(self): # Start with a 200 block chain assert_equal(self.nodes[0].getblockcount(), 200) @@ -40,17 +44,21 @@ class MempoolCoinbaseTest(BitcoinTestFramework): # 3. Indirect (coinbase and child both in chain) : spend_103 and spend_103_1 # Use invalidatblock to make all of the above coinbase spends invalid (immature coinbase), # and make sure the mempool code behaves correctly. - b = [ self.nodes[0].getblockhash(n) for n in range(101, 105) ] - coinbase_txids = [ self.nodes[0].getblock(h)['tx'][0] for h in b ] + b = [self.nodes[0].getblockhash(n) for n in range(101, 105)] + coinbase_txids = [self.nodes[0].getblock(h)['tx'][0] for h in b] spend_101_raw = create_raw_transaction(self.nodes[0], coinbase_txids[1], node1_address, amount=49.99) spend_102_raw = create_raw_transaction(self.nodes[0], coinbase_txids[2], node0_address, amount=49.99) spend_103_raw = create_raw_transaction(self.nodes[0], coinbase_txids[3], node0_address, amount=49.99) # Create a transaction which is time-locked to two blocks in the future - timelock_tx = self.nodes[0].createrawtransaction([{"txid": coinbase_txids[0], "vout": 0}], {node0_address: 49.99}) - # Set the time lock - timelock_tx = timelock_tx.replace("ffffffff", "11111191", 1) - timelock_tx = timelock_tx[:-8] + hex(self.nodes[0].getblockcount() + 2)[2:] + "000000" + timelock_tx = self.nodes[0].createrawtransaction( + inputs=[{ + "txid": coinbase_txids[0], + "vout": 0, + }], + outputs={node0_address: 49.99}, + locktime=self.nodes[0].getblockcount() + 2, + ) timelock_tx = self.nodes[0].signrawtransactionwithwallet(timelock_tx)["hex"] # This will raise an exception because the timelock transaction is too immature to spend assert_raises_rpc_error(-26, "non-final", self.nodes[0].sendrawtransaction, timelock_tx) @@ -69,6 +77,10 @@ class MempoolCoinbaseTest(BitcoinTestFramework): # Broadcast and mine 103_1: spend_103_1_id = self.nodes[0].sendrawtransaction(spend_103_1_raw) last_block = self.nodes[0].generate(1) + # Sync blocks, so that peer 1 gets the block before timelock_tx + # Otherwise, peer 1 would put the timelock_tx in recentRejects + self.sync_all() + # Time-locked transaction can now be spent timelock_tx_id = self.nodes[0].sendrawtransaction(timelock_tx) @@ -76,9 +88,8 @@ class MempoolCoinbaseTest(BitcoinTestFramework): spend_101_id = self.nodes[0].sendrawtransaction(spend_101_raw) spend_102_1_id = self.nodes[0].sendrawtransaction(spend_102_1_raw) - self.sync_all(timeout=720) - assert_equal(set(self.nodes[0].getrawmempool()), {spend_101_id, spend_102_1_id, timelock_tx_id}) + self.sync_all() for node in self.nodes: node.invalidateblock(last_block[0]) @@ -91,10 +102,9 @@ class MempoolCoinbaseTest(BitcoinTestFramework): for node in self.nodes: node.invalidateblock(new_blocks[0]) - self.sync_all(timeout=720) - # mempool should be empty. assert_equal(set(self.nodes[0].getrawmempool()), set()) + self.sync_all() if __name__ == '__main__': diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 8262e30592..1bda167c87 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mining RPCs diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py new file mode 100755 index 0000000000..6046237101 --- /dev/null +++ b/test/functional/p2p_addr_relay.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test addr relay +""" + +from test_framework.messages import ( + CAddress, + NODE_NETWORK, + NODE_WITNESS, + msg_addr, +) +from test_framework.mininode import ( + P2PInterface, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) +import time + +ADDRS = [] +for i in range(10): + addr = CAddress() + addr.time = int(time.time()) + i + addr.nServices = NODE_NETWORK | NODE_WITNESS + addr.ip = "123.123.123.{}".format(i % 256) + addr.port = 8333 + i + ADDRS.append(addr) + + +class AddrReceiver(P2PInterface): + def on_addr(self, message): + for addr in message.addrs: + assert_equal(addr.nServices, 9) + assert addr.ip.startswith('123.123.123.') + assert (8333 <= addr.port < 8343) + + +class AddrTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + + def run_test(self): + self.log.info('Create connection that sends addr messages') + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + msg = msg_addr() + + self.log.info('Send too large addr message') + msg.addrs = ADDRS * 101 + with self.nodes[0].assert_debug_log(['message addr size() = 1010']): + addr_source.send_and_ping(msg) + + self.log.info('Check that addr message content is relayed and added to addrman') + addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) + msg.addrs = ADDRS + with self.nodes[0].assert_debug_log([ + 'Added 10 addresses from 127.0.0.1: 0 tried', + 'received: addr (301 bytes) peer=0', + 'sending addr (301 bytes) peer=1', + ]): + addr_source.send_and_ping(msg) + self.nodes[0].setmocktime(int(time.time()) + 30 * 60) + addr_receiver.sync_with_ping() + + +if __name__ == '__main__': + AddrTest().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 7905cf5018..66e6f8c424 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -378,8 +378,6 @@ class CompactBlocksTest(BitcoinTestFramework): # request for announce in ["inv", "header"]: block = self.build_block_on_tip(node, segwit=segwit) - with mininode_lock: - test_node.last_message.pop("getdata", None) if announce == "inv": test_node.send_message(msg_inv([CInv(2, block.sha256)])) @@ -387,10 +385,8 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) - wait_until(lambda: "getdata" in test_node.last_message, timeout=30, lock=mininode_lock) - assert_equal(len(test_node.last_message["getdata"].inv), 1) + test_node.wait_for_getdata([block.sha256], timeout=30) assert_equal(test_node.last_message["getdata"].inv[0].type, 4) - assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Send back a compactblock message that omits the coinbase comp_block = HeaderAndShortIDs() @@ -567,10 +563,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # We should receive a getdata request - wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) - assert_equal(len(test_node.last_message["getdata"].inv), 1) + test_node.wait_for_getdata([block.sha256], timeout=10) assert test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG - assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Deliver the block if version == 2: diff --git a/test/functional/p2p_dos_header_tree.py b/test/functional/p2p_dos_header_tree.py index 28dd809ca5..f8552cf53d 100755 --- a/test/functional/p2p_dos_header_tree.py +++ b/test/functional/p2p_dos_header_tree.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that we reject low difficulty headers to prevent our block tree from filling up with useless bloat""" diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 188b130a57..a8b768c144 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -7,17 +7,18 @@ Test BIP 37 """ from test_framework.messages import ( + CInv, + MAX_BLOOM_FILTER_SIZE, + MAX_BLOOM_HASH_FUNCS, MSG_BLOCK, MSG_FILTERED_BLOCK, - msg_getdata, - msg_filterload, msg_filteradd, msg_filterclear, + msg_filterload, + msg_getdata, ) -from test_framework.mininode import ( - P2PInterface, - mininode_lock, -) +from test_framework.mininode import P2PInterface +from test_framework.script import MAX_SCRIPT_ELEMENT_SIZE from test_framework.test_framework import BitcoinTestFramework @@ -38,8 +39,9 @@ class FilterNode(P2PInterface): for i in message.inv: # inv messages can only contain TX or BLOCK, so translate BLOCK to FILTERED_BLOCK if i.type == MSG_BLOCK: - i.type = MSG_FILTERED_BLOCK - want.inv.append(i) + want.inv.append(CInv(MSG_FILTERED_BLOCK, i.hash)) + else: + want.inv.append(i) if len(want.inv): self.send_message(want) @@ -63,24 +65,32 @@ class FilterTest(BitcoinTestFramework): self.skip_if_no_wallet() def run_test(self): - self.log.info('Add filtered P2P connection to the node') filter_node = self.nodes[0].add_p2p_connection(FilterNode()) + + self.log.info('Check that too large filter is rejected') + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS+1)) + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xbb'*(MAX_BLOOM_FILTER_SIZE+1))) + + self.log.info('Check that too large data element to add to the filter is rejected') + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filteradd(data=b'\xcc'*(MAX_SCRIPT_ELEMENT_SIZE+1))) + + self.log.info('Add filtered P2P connection to the node') filter_node.send_and_ping(filter_node.watch_filter_init) filter_address = self.nodes[0].decodescript(filter_node.watch_script_pubkey)['addresses'][0] self.log.info('Check that we receive merkleblock and tx if the filter matches a tx in a block') - filter_node.merkleblock_received = False block_hash = self.nodes[0].generatetoaddress(1, filter_address)[0] txid = self.nodes[0].getblock(block_hash)['tx'][0] + filter_node.wait_for_merkleblock(block_hash) filter_node.wait_for_tx(txid) - assert filter_node.merkleblock_received self.log.info('Check that we only receive a merkleblock if the filter does not match a tx in a block') - with mininode_lock: - filter_node.last_message.pop("merkleblock", None) filter_node.tx_received = False - self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress()) - filter_node.wait_for_merkleblock() + block_hash = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0] + filter_node.wait_for_merkleblock(block_hash) assert not filter_node.tx_received self.log.info('Check that we not receive a tx if the filter does not match a mempool tx') @@ -104,6 +114,20 @@ class FilterTest(BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 7) filter_node.wait_for_tx(txid) + self.log.info('Check that request for filtered blocks is ignored if no filter is set') + filter_node.merkleblock_received = False + filter_node.tx_received = False + with self.nodes[0].assert_debug_log(expected_msgs=['received getdata']): + block_hash = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0] + filter_node.wait_for_inv([CInv(MSG_BLOCK, int(block_hash, 16))]) + filter_node.sync_with_ping() + assert not filter_node.merkleblock_received + assert not filter_node.tx_received + + self.log.info('Check that sending "filteradd" if no filter is set is treated as misbehavior') + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filteradd(data=b'letsmisbehave')) + self.log.info("Check that division-by-zero remote crash bug [CVE-2013-5700] is fixed") filter_node.send_and_ping(msg_filterload(data=b'', nHashFuncs=1)) filter_node.send_and_ping(msg_filteradd(data=b'letstrytocrashthisnode')) diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py index fab0887197..c9fbb830c8 100755 --- a/test/functional/p2p_fingerprint.py +++ b/test/functional/p2p_fingerprint.py @@ -90,7 +90,7 @@ class P2PFingerprintTest(BitcoinTestFramework): # Force reorg to a longer chain node0.send_message(msg_headers(new_blocks)) - node0.wait_for_getdata() + node0.wait_for_getdata([x.sha256 for x in new_blocks]) for block in new_blocks: node0.send_and_ping(msg_block(block)) diff --git a/test/functional/p2p_invalid_block.py b/test/functional/p2p_invalid_block.py index 801407757f..e280a62997 100755 --- a/test/functional/p2p_invalid_block.py +++ b/test/functional/p2p_invalid_block.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid blocks. diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index 33b7060060..0155eb21f0 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid locators. @@ -34,7 +34,7 @@ class InvalidLocatorTest(BitcoinTestFramework): msg.locator.vHave = [int(node.getblockhash(i - 1), 16) for i in range(block_count, block_count - (MAX_LOCATOR_SZ), -1)] node.p2p.send_message(msg) if type(msg) == msg_getheaders: - node.p2p.wait_for_header(int(node.getbestblockhash(), 16)) + node.p2p.wait_for_header(node.getbestblockhash()) else: node.p2p.wait_for_block(int(node.getbestblockhash(), 16)) diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 759f111e69..4bd832e8f7 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" @@ -8,14 +8,18 @@ import struct import sys from test_framework import messages -from test_framework.mininode import P2PDataStore, NetworkThread +from test_framework.mininode import ( + NetworkThread, + P2PDataStore, + P2PInterface, +) from test_framework.test_framework import BitcoinTestFramework class msg_unrecognized: """Nonsensical message. Modeled after similar types in test_framework.messages.""" - command = b'badmsg' + msgtype = b'badmsg' def __init__(self, *, str_data): self.str_data = str_data.encode() if not isinstance(str_data, bytes) else str_data @@ -24,7 +28,7 @@ class msg_unrecognized: return messages.ser_string(self.str_data) def __repr__(self): - return "{}(data={})".format(self.command, self.str_data) + return "{}(data={})".format(self.msgtype, self.str_data) class InvalidMessagesTest(BitcoinTestFramework): @@ -46,7 +50,8 @@ class InvalidMessagesTest(BitcoinTestFramework): self.test_magic_bytes() self.test_checksum() self.test_size() - self.test_command() + self.test_msgtype() + self.test_large_inv() node = self.nodes[0] self.node = node @@ -163,7 +168,7 @@ class InvalidMessagesTest(BitcoinTestFramework): msg = conn.build_message(msg_unrecognized(str_data="d")) cut_len = ( 4 + # magic - 12 + # command + 12 + # msgtype 4 #len ) # modify checksum @@ -178,7 +183,7 @@ class InvalidMessagesTest(BitcoinTestFramework): msg = conn.build_message(msg_unrecognized(str_data="d")) cut_len = ( 4 + # magic - 12 # command + 12 # msgtype ) # modify len to MAX_SIZE + 1 msg = msg[:cut_len] + struct.pack("<I", 0x02000000 + 1) + msg[cut_len + 4:] @@ -186,18 +191,31 @@ class InvalidMessagesTest(BitcoinTestFramework): conn.wait_for_disconnect(timeout=1) self.nodes[0].disconnect_p2ps() - def test_command(self): + def test_msgtype(self): conn = self.nodes[0].add_p2p_connection(P2PDataStore()) with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: ERRORS IN HEADER']): msg = msg_unrecognized(str_data="d") - msg.command = b'\xff' * 12 + msg.msgtype = b'\xff' * 12 msg = conn.build_message(msg) - # Modify command + # Modify msgtype msg = msg[:7] + b'\x00' + msg[7 + 1:] self.nodes[0].p2p.send_raw_message(msg) conn.sync_with_ping(timeout=1) self.nodes[0].disconnect_p2ps() + def test_large_inv(self): + conn = self.nodes[0].add_p2p_connection(P2PInterface()) + with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (0 -> 20): message inv size() = 50001']): + msg = messages.msg_inv([messages.CInv(1, 1)] * 50001) + conn.send_and_ping(msg) + with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (20 -> 40): message getdata size() = 50001']): + msg = messages.msg_getdata([messages.CInv(1, 1)] * 50001) + conn.send_and_ping(msg) + with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (40 -> 60): headers message size = 2001']): + msg = messages.msg_headers([messages.CBlockHeader()] * 2001) + conn.send_and_ping(msg) + self.nodes[0].disconnect_p2ps() + def _tweak_msg_data_size(self, message, wrong_size): """ Return a raw message based on another message but with an incorrect data size in diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 5975a52b2a..c70a892463 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid transactions. @@ -118,6 +118,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): tx_orphan_2_invalid = CTransaction() tx_orphan_2_invalid.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_1.sha256, 2))) tx_orphan_2_invalid.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_2_invalid.calc_sha256() self.log.info('Send the orphans ... ') # Send valid orphan txs from p2ps[0] @@ -148,6 +149,22 @@ class InvalidTxRequestTest(BitcoinTestFramework): wait_until(lambda: 1 == len(node.getpeerinfo()), timeout=12) # p2ps[1] is no longer connected assert_equal(expected_mempool, set(node.getrawmempool())) + self.log.info('Test orphan pool overflow') + orphan_tx_pool = [CTransaction() for _ in range(101)] + for i in range(len(orphan_tx_pool)): + orphan_tx_pool[i].vin.append(CTxIn(outpoint=COutPoint(i, 333))) + orphan_tx_pool[i].vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + + with node.assert_debug_log(['mapOrphan overflow, removed 1 tx']): + node.p2p.send_txs_and_test(orphan_tx_pool, node, success=False) + + rejected_parent = CTransaction() + rejected_parent.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_2_invalid.sha256, 0))) + rejected_parent.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + rejected_parent.rehash() + with node.assert_debug_log(['not keeping orphan with rejected parents {}'.format(rejected_parent.hash)]): + node.p2p.send_txs_and_test([rejected_parent], node, success=False) + if __name__ == '__main__': InvalidTxRequestTest().main() diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 2751ae6a5b..7f7430d04e 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test message sending before handshake completion. -A node should never send anything other than VERSION/VERACK/REJECT until it's +A node should never send anything other than VERSION/VERACK until it's received a VERACK. This test connects to a node and sends it a few messages, trying to entice it @@ -12,10 +12,19 @@ into sending us something it shouldn't.""" import time -from test_framework.messages import msg_getaddr, msg_ping, msg_verack +from test_framework.messages import ( + msg_getaddr, + msg_ping, + msg_verack, + msg_version, +) from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import wait_until +from test_framework.util import ( + assert_equal, + assert_greater_than_or_equal, + wait_until, +) banscore = 10 @@ -28,14 +37,13 @@ class CLazyNode(P2PInterface): def bad_message(self, message): self.unexpected_msg = True - self.log.info("should not have received message: %s" % message.command) + self.log.info("should not have received message: %s" % message.msgtype) def on_open(self): self.ever_connected = True def on_version(self, message): self.bad_message(message) def on_verack(self, message): self.bad_message(message) - def on_reject(self, message): self.bad_message(message) def on_inv(self, message): self.bad_message(message) def on_addr(self, message): self.bad_message(message) def on_getdata(self, message): self.bad_message(message) @@ -65,8 +73,6 @@ class CNodeNoVersionBan(CLazyNode): for i in range(banscore): self.send_message(msg_verack()) - def on_reject(self, message): pass - # Node that never sends a version. This one just sits idle and hopes to receive # any message (it shouldn't!) class CNodeNoVersionIdle(CLazyNode): @@ -79,7 +85,6 @@ class CNodeNoVerackIdle(CLazyNode): self.version_received = False super().__init__() - def on_reject(self, message): pass def on_verack(self, message): pass # When version is received, don't reply with a verack. Instead, see if the # node will give us a message that it shouldn't. This is not an exhaustive @@ -90,6 +95,14 @@ class CNodeNoVerackIdle(CLazyNode): self.send_message(msg_getaddr()) +class P2PVersionStore(P2PInterface): + version_received = None + + def on_version(self, msg): + super().on_version(msg) + self.version_received = msg + + class P2PLeakTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -127,6 +140,27 @@ class P2PLeakTest(BitcoinTestFramework): assert no_version_idlenode.unexpected_msg == False assert no_verack_idlenode.unexpected_msg == False + self.log.info('Check that the version message does not leak the local address of the node') + time_begin = int(time.time()) + p2p_version_store = self.nodes[0].add_p2p_connection(P2PVersionStore()) + time_end = time.time() + ver = p2p_version_store.version_received + assert_greater_than_or_equal(ver.nTime, time_begin) + assert_greater_than_or_equal(time_end, ver.nTime) + assert_equal(ver.addrFrom.port, 0) + assert_equal(ver.addrFrom.ip, '0.0.0.0') + assert_equal(ver.nStartingHeight, 201) + assert_equal(ver.nRelay, 1) + + self.log.info('Check that old nodes are disconnected') + p2p_old_node = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) + old_version_msg = msg_version() + old_version_msg.nVersion = 31799 + wait_until(lambda: p2p_old_node.is_connected) + with self.nodes[0].assert_debug_log(['peer=4 using obsolete version 31799; disconnecting']): + p2p_old_node.send_message(old_version_msg) + p2p_old_node.wait_for_disconnect() + if __name__ == '__main__': P2PLeakTest().main() diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index c6fcc5e200..6b3436fa5f 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that we don't leak txs to inbound peers that we haven't yet announced to""" diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 3a7bf4bfc3..2c200fccad 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test p2p permission message. diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index d8dce7fe56..dbdce6552a 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test segwit transactions and blocks on P2P network.""" @@ -159,7 +159,7 @@ class TestP2PConn(P2PInterface): self.last_message.pop("getdata", None) self.send_message(msg_inv(inv=[CInv(1, tx.sha256)])) if success: - self.wait_for_getdata(timeout) + self.wait_for_getdata([tx.sha256], timeout) else: time.sleep(timeout) assert not self.last_message.get("getdata") @@ -176,7 +176,7 @@ class TestP2PConn(P2PInterface): self.send_message(msg_inv(inv=[CInv(2, block.sha256)])) self.wait_for_getheaders() self.send_message(msg) - self.wait_for_getdata() + self.wait_for_getdata([block.sha256]) def request_block(self, blockhash, inv_type, timeout=60): with mininode_lock: @@ -862,13 +862,13 @@ class SegWitTest(BitcoinTestFramework): # We can't send over the p2p network, because this is too big to relay # TODO: repeat this test with a block that can be relayed - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal('bad-witness-nonce-size', self.nodes[0].submitblock(block.serialize().hex())) assert self.nodes[0].getbestblockhash() != block.hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() assert get_virtual_size(block) < MAX_BLOCK_BASE_SIZE - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(None, self.nodes[0].submitblock(block.serialize().hex())) assert self.nodes[0].getbestblockhash() == block.hash @@ -975,14 +975,14 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block, nonce=1) block.vtx[0].wit = CTxWitness() # drop the nonce block.solve() - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal('bad-witness-merkle-match', self.nodes[0].submitblock(block.serialize().hex())) assert self.nodes[0].getbestblockhash() != block.hash # Now redo commitment with the standard nonce, but let bitcoind fill it in. add_witness_commitment(block, nonce=0) block.vtx[0].wit = CTxWitness() block.solve() - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(None, self.nodes[0].submitblock(block.serialize().hex())) assert_equal(self.nodes[0].getbestblockhash(), block.hash) # This time, add a tx with non-empty witness, but don't supply @@ -997,7 +997,7 @@ class SegWitTest(BitcoinTestFramework): block_2.vtx[0].vout.pop() block_2.vtx[0].wit = CTxWitness() - self.nodes[0].submitblock(block_2.serialize().hex()) + assert_equal('bad-txnmrklroot', self.nodes[0].submitblock(block_2.serialize().hex())) # Tip should not advance! assert self.nodes[0].getbestblockhash() != block_2.hash diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 84d818400a..a8fba306a7 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of headers messages to announce blocks. @@ -144,13 +144,6 @@ class BaseNode(P2PInterface): getblocks_message.locator.vHave = locator self.send_message(getblocks_message) - def wait_for_getdata(self, hash_list, timeout=60): - if hash_list == []: - return - - test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list - wait_until(test_function, timeout=timeout, lock=mininode_lock) - def wait_for_block_announcement(self, block_hash, timeout=60): test_function = lambda: self.last_blockhash_announced == block_hash wait_until(test_function, timeout=timeout, lock=mininode_lock) diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index c1235f8a6b..5a4fa42988 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various net timeouts. diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index b56dc994e7..a999fba818 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ @@ -152,6 +152,10 @@ class TxDownloadTest(BitcoinTestFramework): wait_until(lambda: p.tx_getdata_count == MAX_GETDATA_IN_FLIGHT + 2) self.nodes[0].setmocktime(0) + def test_spurious_notfound(self): + self.log.info('Check that spurious notfound is ignored') + self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(1, 1)])) + def run_test(self): # Setup the p2p connections self.peers = [] @@ -161,6 +165,8 @@ class TxDownloadTest(BitcoinTestFramework): self.log.info("Nodes are setup with {} incoming connections each".format(NUM_INBOUND)) + self.test_spurious_notfound() + # Test the in-flight max first, because we want no transactions in # flight ahead of this test. self.test_in_flight_max() diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index d18d4b069a..3aaf4b9977 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index a6eaaa4539..6273c229ae 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPCs related to blockchainstate. diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index a983716177..56e9ecfcc2 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -4,13 +4,14 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multisig RPCs""" +from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create, drop_origins from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_raises_rpc_error, assert_equal, ) -from test_framework.key import ECPubKey +from test_framework.key import ECPubKey, ECKey, bytes_to_wif import binascii import decimal @@ -28,10 +29,14 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): self.skip_if_no_wallet() def get_keys(self): + self.pub = [] + self.priv = [] node0, node1, node2 = self.nodes - add = [node1.getnewaddress() for _ in range(self.nkeys)] - self.pub = [node1.getaddressinfo(a)["pubkey"] for a in add] - self.priv = [node1.dumpprivkey(a) for a in add] + for _ in range(self.nkeys): + k = ECKey() + k.generate() + self.pub.append(k.get_pubkey().get_bytes().hex()) + self.priv.append(bytes_to_wif(k.get_bytes(), k.is_compressed)) self.final = node2.getnewaddress() def run_test(self): @@ -64,17 +69,20 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): pk_obj.compressed = False pk2 = binascii.hexlify(pk_obj.get_bytes()).decode() + node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) + wmulti0 = node0.get_wallet_rpc('wmulti0') + # Check all permutations of keys because order matters apparently for keys in itertools.permutations([pk0, pk1, pk2]): # Results should be the same as this legacy one legacy_addr = node0.createmultisig(2, keys, 'legacy')['address'] - assert_equal(legacy_addr, node0.addmultisigaddress(2, keys, '', 'legacy')['address']) + assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '', 'legacy')['address']) # Generate addresses with the segwit types. These should all make legacy addresses - assert_equal(legacy_addr, node0.createmultisig(2, keys, 'bech32')['address']) - assert_equal(legacy_addr, node0.createmultisig(2, keys, 'p2sh-segwit')['address']) - assert_equal(legacy_addr, node0.addmultisigaddress(2, keys, '', 'bech32')['address']) - assert_equal(legacy_addr, node0.addmultisigaddress(2, keys, '', 'p2sh-segwit')['address']) + assert_equal(legacy_addr, wmulti0.createmultisig(2, keys, 'bech32')['address']) + assert_equal(legacy_addr, wmulti0.createmultisig(2, keys, 'p2sh-segwit')['address']) + assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '', 'bech32')['address']) + assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '', 'p2sh-segwit')['address']) self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: @@ -89,6 +97,8 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): assert_equal(self.nodes[0].deriveaddresses(sorted_key_desc)[0], t['address']) def check_addmultisigaddress_errors(self): + if self.options.descriptors: + return self.log.info('Check that addmultisigaddress fails when the private keys are missing') addresses = [self.nodes[1].getnewaddress(address_type='legacy') for _ in range(2)] assert_raises_rpc_error(-5, 'no full public key for address', lambda: self.nodes[0].addmultisigaddress(nrequired=1, keys=addresses)) @@ -115,6 +125,15 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): def do_multisig(self): node0, node1, node2 = self.nodes + if 'wmulti' not in node1.listwallets(): + try: + node1.loadwallet('wmulti') + except JSONRPCException as e: + if e.error['code'] == -18 and 'Wallet wmulti not found' in e.error['message']: + node1.createwallet(wallet_name='wmulti', disable_private_keys=True) + else: + raise + wmulti = node1.get_wallet_rpc('wmulti') # Construct the expected descriptor desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub)) @@ -134,7 +153,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): assert madd[0:4] == "bcrt" # actually a bech32 address # compare against addmultisigaddress - msigw = node1.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) + msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) maddw = msigw["address"] mredeemw = msigw["redeemScript"] assert_equal(desc, drop_origins(msigw['descriptor'])) @@ -194,6 +213,8 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): txinfo = node0.getrawtransaction(tx, True, blk) self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) + wmulti.unloadwallet() + if __name__ == '__main__': RpcCreateMultiSigTest().main() diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 438e7f68e0..e65787ce08 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the generation of UTXO snapshots using `dumptxoutset`. diff --git a/test/functional/rpc_estimatefee.py b/test/functional/rpc_estimatefee.py index 8bdecfc8cd..1fff9e1512 100755 --- a/test/functional/rpc_estimatefee.py +++ b/test/functional/rpc_estimatefee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the estimatefee RPCs. diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index c435ef24ce..4bc4913bda 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the fundrawtransaction RPC.""" @@ -500,11 +500,16 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[1].getnewaddress() self.nodes[1].getrawchangeaddress() inputs = [] - outputs = {self.nodes[0].getnewaddress():1.1} + outputs = {self.nodes[0].getnewaddress():1.09999500} rawtx = self.nodes[1].createrawtransaction(inputs, outputs) + # fund a transaction that does not require a new key for the change output + self.nodes[1].fundrawtransaction(rawtx) + # fund a transaction that requires a new key for the change output # creating the key must be impossible because the wallet is locked - assert_raises_rpc_error(-4, "Keypool ran out, please call keypoolrefill first", self.nodes[1].fundrawtransaction, rawtx) + outputs = {self.nodes[0].getnewaddress():1.1} + rawtx = self.nodes[1].createrawtransaction(inputs, outputs) + assert_raises_rpc_error(-4, "Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.", self.nodes[1].fundrawtransaction, rawtx) # Refill the keypool. self.nodes[1].walletpassphrase("test", 100) diff --git a/test/functional/rpc_generateblock.py b/test/functional/rpc_generateblock.py new file mode 100755 index 0000000000..aa58c0af9d --- /dev/null +++ b/test/functional/rpc_generateblock.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +'''Test generateblock rpc. +''' + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + + +class GenerateBlockTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + node = self.nodes[0] + + self.log.info('Generate an empty block to address') + address = node.getnewaddress() + hash = node.generateblock(output=address, transactions=[])['hash'] + block = node.getblock(blockhash=hash, verbose=2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], address) + + self.log.info('Generate an empty block to a descriptor') + hash = node.generateblock('addr(' + address + ')', [])['hash'] + block = node.getblock(blockhash=hash, verbosity=2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], address) + + self.log.info('Generate an empty block to a combo descriptor with compressed pubkey') + combo_key = '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' + combo_address = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' + hash = node.generateblock('combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], combo_address) + + self.log.info('Generate an empty block to a combo descriptor with uncompressed pubkey') + combo_key = '0408ef68c46d20596cc3f6ddf7c8794f71913add807f1dc55949fa805d764d191c0b7ce6894c126fce0babc6663042f3dde9b0cf76467ea315514e5a6731149c67' + combo_address = 'mkc9STceoCcjoXEXe6cm66iJbmjM6zR9B2' + hash = node.generateblock('combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], combo_address) + + # Generate 110 blocks to spend + node.generatetoaddress(110, address) + + # Generate some extra mempool transactions to verify they don't get mined + for i in range(10): + node.sendtoaddress(address, 0.001) + + self.log.info('Generate block with txid') + txid = node.sendtoaddress(address, 1) + hash = node.generateblock(address, [txid])['hash'] + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + assert_equal(block['tx'][1], txid) + + self.log.info('Generate block with raw tx') + utxos = node.listunspent(addresses=[address]) + raw = node.createrawtransaction([{'txid':utxos[0]['txid'], 'vout':utxos[0]['vout']}],[{address:1}]) + signed_raw = node.signrawtransactionwithwallet(raw)['hex'] + hash = node.generateblock(address, [signed_raw])['hash'] + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + txid = block['tx'][1] + assert_equal(node.gettransaction(txid)['hex'], signed_raw) + + self.log.info('Fail to generate block with out of order txs') + raw1 = node.createrawtransaction([{'txid':txid, 'vout':0}],[{address:0.9999}]) + signed_raw1 = node.signrawtransactionwithwallet(raw1)['hex'] + txid1 = node.sendrawtransaction(signed_raw1) + raw2 = node.createrawtransaction([{'txid':txid1, 'vout':0}],[{address:0.999}]) + signed_raw2 = node.signrawtransactionwithwallet(raw2)['hex'] + assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', node.generateblock, address, [signed_raw2, txid1]) + + self.log.info('Fail to generate block with txid not in mempool') + missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' + assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', node.generateblock, address, [missing_txid]) + + self.log.info('Fail to generate block with invalid raw tx') + invalid_raw_tx = '0000' + assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, node.generateblock, address, [invalid_raw_tx]) + + self.log.info('Fail to generate block with invalid address/descriptor') + assert_raises_rpc_error(-5, 'Invalid address or descriptor', node.generateblock, '1234', []) + + self.log.info('Fail to generate block with a ranged descriptor') + ranged_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0/*)' + assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', node.generateblock, ranged_descriptor, []) + + self.log.info('Fail to generate block with a descriptor missing a private key') + child_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0\'/0)' + assert_raises_rpc_error(-5, 'Cannot derive script without private keys', node.generateblock, child_descriptor, []) + +if __name__ == '__main__': + GenerateBlockTest().main() diff --git a/test/functional/rpc_getaddressinfo_label_deprecation.py b/test/functional/rpc_getaddressinfo_label_deprecation.py index 5e739ebede..4c6b2fe5cf 100755 --- a/test/functional/rpc_getaddressinfo_label_deprecation.py +++ b/test/functional/rpc_getaddressinfo_label_deprecation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py b/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py index 3f2e8dee18..903f5536b9 100755 --- a/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py +++ b/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020-2019 The Bitcoin Core developers +# Copyright (c) 2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 3a63377545..51d136d26a 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the Partially Signed Transaction RPCs. @@ -48,18 +48,23 @@ class PSBTTest(BitcoinTestFramework): disconnect_nodes(offline_node, 2) disconnect_nodes(mining_node, 0) + # Create watchonly on online_node + online_node.createwallet(wallet_name='wonline', disable_private_keys=True) + wonline = online_node.get_wallet_rpc('wonline') + w2 = online_node.get_wallet_rpc('') + # Mine a transaction that credits the offline address offline_addr = offline_node.getnewaddress(address_type="p2sh-segwit") - online_addr = online_node.getnewaddress(address_type="p2sh-segwit") - online_node.importaddress(offline_addr, "", False) + online_addr = w2.getnewaddress(address_type="p2sh-segwit") + wonline.importaddress(offline_addr, "", False) mining_node.sendtoaddress(address=offline_addr, amount=1.0) mining_node.generate(nblocks=1) self.sync_blocks([mining_node, online_node]) # Construct an unsigned PSBT on the online node (who doesn't know the output is Segwit, so will include a non-witness UTXO) - utxos = online_node.listunspent(addresses=[offline_addr]) - raw = online_node.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}]) - psbt = online_node.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"] + utxos = wonline.listunspent(addresses=[offline_addr]) + raw = wonline.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}]) + psbt = wonline.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"] assert "non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0] # Have the offline node sign the PSBT (which will update the UTXO to segwit) @@ -72,6 +77,8 @@ class PSBTTest(BitcoinTestFramework): self.sync_blocks([mining_node, online_node]) assert_equal(online_node.gettxout(txid,0)["confirmations"], 1) + wonline.unloadwallet() + # Reconnect connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) @@ -89,13 +96,23 @@ class PSBTTest(BitcoinTestFramework): final_tx = self.nodes[0].finalizepsbt(signed_tx)['hex'] self.nodes[0].sendrawtransaction(final_tx) - # Create p2sh, p2wpkh, and p2wsh addresses + # Get pubkeys pubkey0 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress())['pubkey'] pubkey1 = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['pubkey'] pubkey2 = self.nodes[2].getaddressinfo(self.nodes[2].getnewaddress())['pubkey'] - p2sh = self.nodes[1].addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "legacy")['address'] - p2wsh = self.nodes[1].addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "bech32")['address'] - p2sh_p2wsh = self.nodes[1].addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "p2sh-segwit")['address'] + + # Setup watchonly wallets + self.nodes[2].createwallet(wallet_name='wmulti', disable_private_keys=True) + wmulti = self.nodes[2].get_wallet_rpc('wmulti') + + # Create all the addresses + p2sh = wmulti.addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "legacy")['address'] + p2wsh = wmulti.addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "bech32")['address'] + p2sh_p2wsh = wmulti.addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "p2sh-segwit")['address'] + if not self.options.descriptors: + wmulti.importaddress(p2sh) + wmulti.importaddress(p2wsh) + wmulti.importaddress(p2sh_p2wsh) p2wpkh = self.nodes[1].getnewaddress("", "bech32") p2pkh = self.nodes[1].getnewaddress("", "legacy") p2sh_p2wpkh = self.nodes[1].getnewaddress("", "p2sh-segwit") @@ -146,11 +163,14 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Fee exceeds maximum configured by -maxtxfee", self.nodes[1].walletcreatefundedpsbt, [{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99}, 0, {"feeRate": 10}) # partially sign multisig things with node 1 - psbtx = self.nodes[1].walletcreatefundedpsbt([{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], {self.nodes[1].getnewaddress():29.99})['psbt'] + psbtx = wmulti.walletcreatefundedpsbt(inputs=[{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], outputs={self.nodes[1].getnewaddress():29.99}, options={'changeAddress': self.nodes[1].getrawchangeaddress()})['psbt'] walletprocesspsbt_out = self.nodes[1].walletprocesspsbt(psbtx) psbtx = walletprocesspsbt_out['psbt'] assert_equal(walletprocesspsbt_out['complete'], False) + # Unload wmulti, we don't need it anymore + wmulti.unloadwallet() + # partially sign with node 2. This should be complete and sendable walletprocesspsbt_out = self.nodes[2].walletprocesspsbt(psbtx) assert_equal(walletprocesspsbt_out['complete'], True) @@ -297,7 +317,7 @@ class PSBTTest(BitcoinTestFramework): # Signer tests for i, signer in enumerate(signers): - self.nodes[2].createwallet("wallet{}".format(i)) + self.nodes[2].createwallet(wallet_name="wallet{}".format(i)) wrpc = self.nodes[2].get_wallet_rpc("wallet{}".format(i)) for key in signer['privkeys']: wrpc.importprivkey(key) diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index c3d34be0dd..861b394e70 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the scantxoutset rpc call.""" diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index 17686f3a78..3d08202724 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction signing using the signrawtransaction* RPCs.""" @@ -27,6 +27,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): 1) The transaction has a complete set of signatures 2) No script verification error occurred""" + self.log.info("Test valid raw transaction with one input") privKeys = ['cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N', 'cVKpPfVKSJxKqVpE9awvXNWuLHCa5j5tiE7K6zbUSptFpTEtiFrA'] inputs = [ @@ -49,7 +50,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert 'errors' not in rawTxSigned def test_with_lock_outputs(self): - """Test correct error reporting when trying to sign a locked output""" + self.log.info("Test correct error reporting when trying to sign a locked output") self.nodes[0].encryptwallet("password") rawTx = '020000000156b958f78e3f24e0b2f4e4db1255426b0902027cb37e3ddadb52e37c3557dddb0000000000ffffffff01c0a6b929010000001600149a2ee8c77140a053f36018ac8124a6ececc1668a00000000' @@ -65,6 +66,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): 4) Two script verification errors occurred 5) Script verification errors have certain properties ("txid", "vout", "scriptSig", "sequence", "error") 6) The verification errors refer to the invalid (vin 1) and missing input (vin 2)""" + self.log.info("Test script verification errors") privKeys = ['cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N'] inputs = [ @@ -147,7 +149,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert not rawTxSigned['errors'][0]['witness'] def witness_script_test(self): - # Now test signing transaction to P2SH-P2WSH addresses without wallet + self.log.info("Test signing transaction to P2SH-P2WSH addresses without wallet") # Create a new P2SH-P2WSH 1-of-1 multisig address: embedded_address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress()) embedded_privkey = self.nodes[1].dumpprivkey(embedded_address["address"]) @@ -169,29 +171,18 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert 'complete' in spending_tx_signed assert_equal(spending_tx_signed['complete'], True) - self.log.info('Try with a P2PKH script as the witnessScript') - embedded_addr_info = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress('', 'legacy')) - embedded_privkey = self.nodes[1].dumpprivkey(embedded_addr_info['address']) - witness_script = embedded_addr_info['scriptPubKey'] - redeem_script = CScript([OP_0, sha256(check_script(witness_script))]).hex() - addr = script_to_p2sh(redeem_script) - script_pub_key = self.nodes[1].validateaddress(addr)['scriptPubKey'] - # Fund that address - txid = self.nodes[0].sendtoaddress(addr, 10) - vout = find_vout_for_address(self.nodes[0], txid, addr) - self.nodes[0].generate(1) - # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys - spending_tx = self.nodes[0].createrawtransaction([{'txid': txid, 'vout': vout}], {self.nodes[1].getnewaddress(): Decimal("9.999")}) - spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [{'txid': txid, 'vout': vout, 'scriptPubKey': script_pub_key, 'redeemScript': redeem_script, 'witnessScript': witness_script, 'amount': 10}]) - # Check the signing completed successfully - assert 'complete' in spending_tx_signed - assert_equal(spending_tx_signed['complete'], True) - self.nodes[0].sendrawtransaction(spending_tx_signed['hex']) + # Now test with P2PKH and P2PK scripts as the witnessScript + for tx_type in ['P2PKH', 'P2PK']: # these tests are order-independent + self.verify_txn_with_witness_script(tx_type) - self.log.info('Try with a P2PK script as the witnessScript') + def verify_txn_with_witness_script(self, tx_type): + self.log.info("Test with a {} script as the witnessScript".format(tx_type)) embedded_addr_info = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress('', 'legacy')) embedded_privkey = self.nodes[1].dumpprivkey(embedded_addr_info['address']) - witness_script = CScript([hex_str_to_bytes(embedded_addr_info['pubkey']), OP_CHECKSIG]).hex() + witness_script = { + 'P2PKH': embedded_addr_info['scriptPubKey'], + 'P2PK': CScript([hex_str_to_bytes(embedded_addr_info['pubkey']), OP_CHECKSIG]).hex() + }.get(tx_type, "Invalid tx_type") redeem_script = CScript([OP_0, sha256(check_script(witness_script))]).hex() addr = script_to_p2sh(redeem_script) script_pub_key = self.nodes[1].validateaddress(addr)['scriptPubKey'] diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 6a7e91216a..8f410f233e 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Encode and decode BASE58, P2PKH and P2SH addresses.""" diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 4ba6ac1db2..05308931e3 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -101,23 +101,26 @@ class AuthServiceProxy(): if os.name == 'nt': # Windows somehow does not like to re-use connections # TODO: Find out why the connection would disconnect occasionally and make it reusable on Windows + # Avoid "ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine" self._set_conn() try: self.__conn.request(method, path, postdata, headers) return self._get_response() - except http.client.BadStatusLine as e: - if e.line == "''": # if connection was closed, try again + except (BrokenPipeError, ConnectionResetError): + # Python 3.5+ raises BrokenPipeError when the connection was reset + # ConnectionResetError happens on FreeBSD + self.__conn.close() + self.__conn.request(method, path, postdata, headers) + return self._get_response() + except OSError as e: + retry = ( + '[WinError 10053] An established connection was aborted by the software in your host machine' in str(e)) + if retry: self.__conn.close() self.__conn.request(method, path, postdata, headers) return self._get_response() else: raise - except (BrokenPipeError, ConnectionResetError): - # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset - # ConnectionResetError happens on FreeBSD with Python 3.4 - self.__conn.close() - self.__conn.request(method, path, postdata, headers) - return self._get_response() def get_request(self, *args, **argsn): AuthServiceProxy.__id_count += 1 diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 912c0ca978..f2d6fba4a6 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -8,6 +8,8 @@ keys, and is trivially vulnerable to side channel attacks. Do not use for anything but tests.""" import random +from .address import byte_to_base58 + def modinv(a, n): """Compute the modular inverse of a modulo n @@ -384,3 +386,14 @@ class ECKey(): rb = r.to_bytes((r.bit_length() + 8) // 8, 'big') sb = s.to_bytes((s.bit_length() + 8) // 8, 'big') return b'\x30' + bytes([4 + len(rb) + len(sb), 2, len(rb)]) + rb + bytes([2, len(sb)]) + sb + +def bytes_to_wif(b, compressed=True): + if compressed: + b += b'\x01' + return byte_to_base58(b, 239) + +def generate_wif_key(): + # Makes a WIF privkey for imports + k = ECKey() + k.generate() + return bytes_to_wif(k.get_bytes(), k.is_compressed) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 5f8fcc6fd8..4855f62a8f 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2019 The Bitcoin Core developers +# Copyright (c) 2010-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin test framework primitive and message structures @@ -37,6 +37,8 @@ MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version MAX_LOCATOR_SZ = 101 MAX_BLOCK_BASE_SIZE = 1000000 +MAX_BLOOM_FILTER_SIZE = 36000 +MAX_BLOOM_HASH_FUNCS = 50 COIN = 100000000 # 1 btc in satoshis MAX_MONEY = 21000000 * COIN @@ -601,16 +603,16 @@ class CBlock(CBlockHeader): __slots__ = ("vtx",) def __init__(self, header=None): - super(CBlock, self).__init__(header) + super().__init__(header) self.vtx = [] def deserialize(self, f): - super(CBlock, self).deserialize(f) + super().deserialize(f) self.vtx = deser_vector(f, CTransaction) def serialize(self, with_witness=True): r = b"" - r += super(CBlock, self).serialize() + r += super().serialize() if with_witness: r += ser_vector(self.vtx, "serialize_with_witness") else: @@ -750,7 +752,7 @@ class P2PHeaderAndShortIDs: class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs): __slots__ = () def serialize(self): - return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True) + return super().serialize(with_witness=True) # Calculate the BIP 152-compact blocks shortid for a given transaction hash def calculate_shortid(k0, k1, tx_hash): @@ -946,7 +948,7 @@ class CMerkleBlock: class msg_version: __slots__ = ("addrFrom", "addrTo", "nNonce", "nRelay", "nServices", "nStartingHeight", "nTime", "nVersion", "strSubVer") - command = b"version" + msgtype = b"version" def __init__(self): self.nVersion = MY_VERSION @@ -1004,7 +1006,7 @@ class msg_version: class msg_verack: __slots__ = () - command = b"verack" + msgtype = b"verack" def __init__(self): pass @@ -1021,7 +1023,7 @@ class msg_verack: class msg_addr: __slots__ = ("addrs",) - command = b"addr" + msgtype = b"addr" def __init__(self): self.addrs = [] @@ -1038,7 +1040,7 @@ class msg_addr: class msg_inv: __slots__ = ("inv",) - command = b"inv" + msgtype = b"inv" def __init__(self, inv=None): if inv is None: @@ -1058,7 +1060,7 @@ class msg_inv: class msg_getdata: __slots__ = ("inv",) - command = b"getdata" + msgtype = b"getdata" def __init__(self, inv=None): self.inv = inv if inv is not None else [] @@ -1075,7 +1077,7 @@ class msg_getdata: class msg_getblocks: __slots__ = ("locator", "hashstop") - command = b"getblocks" + msgtype = b"getblocks" def __init__(self): self.locator = CBlockLocator() @@ -1099,7 +1101,7 @@ class msg_getblocks: class msg_tx: __slots__ = ("tx",) - command = b"tx" + msgtype = b"tx" def __init__(self, tx=CTransaction()): self.tx = tx @@ -1123,7 +1125,7 @@ class msg_no_witness_tx(msg_tx): class msg_block: __slots__ = ("block",) - command = b"block" + msgtype = b"block" def __init__(self, block=None): if block is None: @@ -1142,12 +1144,12 @@ class msg_block: # for cases where a user needs tighter control over what is sent over the wire -# note that the user must supply the name of the command, and the data +# note that the user must supply the name of the msgtype, and the data class msg_generic: - __slots__ = ("command", "data") + __slots__ = ("msgtype", "data") - def __init__(self, command, data=None): - self.command = command + def __init__(self, msgtype, data=None): + self.msgtype = msgtype self.data = data def serialize(self): @@ -1165,7 +1167,7 @@ class msg_no_witness_block(msg_block): class msg_getaddr: __slots__ = () - command = b"getaddr" + msgtype = b"getaddr" def __init__(self): pass @@ -1182,7 +1184,7 @@ class msg_getaddr: class msg_ping: __slots__ = ("nonce",) - command = b"ping" + msgtype = b"ping" def __init__(self, nonce=0): self.nonce = nonce @@ -1201,7 +1203,7 @@ class msg_ping: class msg_pong: __slots__ = ("nonce",) - command = b"pong" + msgtype = b"pong" def __init__(self, nonce=0): self.nonce = nonce @@ -1220,7 +1222,7 @@ class msg_pong: class msg_mempool: __slots__ = () - command = b"mempool" + msgtype = b"mempool" def __init__(self): pass @@ -1237,7 +1239,7 @@ class msg_mempool: class msg_notfound: __slots__ = ("vec", ) - command = b"notfound" + msgtype = b"notfound" def __init__(self, vec=None): self.vec = vec or [] @@ -1254,7 +1256,7 @@ class msg_notfound: class msg_sendheaders: __slots__ = () - command = b"sendheaders" + msgtype = b"sendheaders" def __init__(self): pass @@ -1275,7 +1277,7 @@ class msg_sendheaders: # hash_stop (hash of last desired block header, 0 to get as many as possible) class msg_getheaders: __slots__ = ("hashstop", "locator",) - command = b"getheaders" + msgtype = b"getheaders" def __init__(self): self.locator = CBlockLocator() @@ -1301,7 +1303,7 @@ class msg_getheaders: # <count> <vector of block headers> class msg_headers: __slots__ = ("headers",) - command = b"headers" + msgtype = b"headers" def __init__(self, headers=None): self.headers = headers if headers is not None else [] @@ -1321,15 +1323,28 @@ class msg_headers: class msg_merkleblock: - command = b"merkleblock" + __slots__ = ("merkleblock",) + msgtype = b"merkleblock" + + def __init__(self, merkleblock=None): + if merkleblock is None: + self.merkleblock = CMerkleBlock() + else: + self.merkleblock = merkleblock def deserialize(self, f): - pass # Placeholder for now + self.merkleblock.deserialize(f) + + def serialize(self): + return self.merkleblock.serialize() + + def __repr__(self): + return "msg_merkleblock(merkleblock=%s)" % (repr(self.merkleblock)) class msg_filterload: __slots__ = ("data", "nHashFuncs", "nTweak", "nFlags") - command = b"filterload" + msgtype = b"filterload" def __init__(self, data=b'00', nHashFuncs=0, nTweak=0, nFlags=0): self.data = data @@ -1358,7 +1373,7 @@ class msg_filterload: class msg_filteradd: __slots__ = ("data") - command = b"filteradd" + msgtype = b"filteradd" def __init__(self, data): self.data = data @@ -1377,7 +1392,7 @@ class msg_filteradd: class msg_filterclear: __slots__ = () - command = b"filterclear" + msgtype = b"filterclear" def __init__(self): pass @@ -1394,7 +1409,7 @@ class msg_filterclear: class msg_feefilter: __slots__ = ("feerate",) - command = b"feefilter" + msgtype = b"feefilter" def __init__(self, feerate=0): self.feerate = feerate @@ -1413,7 +1428,7 @@ class msg_feefilter: class msg_sendcmpct: __slots__ = ("announce", "version") - command = b"sendcmpct" + msgtype = b"sendcmpct" def __init__(self): self.announce = False @@ -1435,7 +1450,7 @@ class msg_sendcmpct: class msg_cmpctblock: __slots__ = ("header_and_shortids",) - command = b"cmpctblock" + msgtype = b"cmpctblock" def __init__(self, header_and_shortids = None): self.header_and_shortids = header_and_shortids @@ -1455,7 +1470,7 @@ class msg_cmpctblock: class msg_getblocktxn: __slots__ = ("block_txn_request",) - command = b"getblocktxn" + msgtype = b"getblocktxn" def __init__(self): self.block_txn_request = None @@ -1475,7 +1490,7 @@ class msg_getblocktxn: class msg_blocktxn: __slots__ = ("block_transactions",) - command = b"blocktxn" + msgtype = b"blocktxn" def __init__(self): self.block_transactions = BlockTransactions() diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index ad330f2a93..6aa73623e6 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2019 The Bitcoin Core developers +# Copyright (c) 2010-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin P2P network half-a-node. @@ -180,7 +180,7 @@ class P2PConnection(asyncio.Protocol): raise ValueError("magic bytes mismatch: {} != {}".format(repr(self.magic_bytes), repr(self.recvbuf))) if len(self.recvbuf) < 4 + 12 + 4 + 4: return - command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] + msgtype = self.recvbuf[4:4+12].split(b"\x00", 1)[0] msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] checksum = self.recvbuf[4+12+4:4+12+4+4] if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen: @@ -191,10 +191,10 @@ class P2PConnection(asyncio.Protocol): if checksum != h[:4]: raise ValueError("got bad checksum " + repr(self.recvbuf)) self.recvbuf = self.recvbuf[4+12+4+4+msglen:] - if command not in MESSAGEMAP: - raise ValueError("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) + if msgtype not in MESSAGEMAP: + raise ValueError("Received unknown msgtype from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, msgtype, repr(msg))) f = BytesIO(msg) - t = MESSAGEMAP[command]() + t = MESSAGEMAP[msgtype]() t.deserialize(f) self._log_message("receive", t) self.on_message(t) @@ -233,11 +233,11 @@ class P2PConnection(asyncio.Protocol): def build_message(self, message): """Build a serialized P2P message""" - command = message.command + msgtype = message.msgtype data = message.serialize() tmsg = self.magic_bytes - tmsg += command - tmsg += b"\x00" * (12 - len(command)) + tmsg += msgtype + tmsg += b"\x00" * (12 - len(msgtype)) tmsg += struct.pack("<I", len(data)) th = sha256(data) h = sha256(th) @@ -304,10 +304,10 @@ class P2PInterface(P2PConnection): and the most recent message of each type.""" with mininode_lock: try: - command = message.command.decode('ascii') - self.message_count[command] += 1 - self.last_message[command] = message - getattr(self, 'on_' + command)(message) + msgtype = message.msgtype.decode('ascii') + self.message_count[msgtype] += 1 + self.last_message[msgtype] = message + getattr(self, 'on_' + msgtype)(message) except: print("ERROR delivering %s (%s)" % (repr(message), sys.exc_info()[0])) raise @@ -339,7 +339,6 @@ class P2PInterface(P2PConnection): def on_merkleblock(self, message): pass def on_notfound(self, message): pass def on_pong(self, message): pass - def on_reject(self, message): pass def on_sendcmpct(self, message): pass def on_sendheaders(self, message): pass def on_tx(self, message): pass @@ -393,32 +392,31 @@ class P2PInterface(P2PConnection): last_headers = self.last_message.get('headers') if not last_headers: return False - return last_headers.headers[0].rehash() == blockhash + return last_headers.headers[0].rehash() == int(blockhash, 16) wait_until(test_function, timeout=timeout, lock=mininode_lock) - def wait_for_merkleblock(self, timeout=60): + def wait_for_merkleblock(self, blockhash, timeout=60): def test_function(): assert self.is_connected last_filtered_block = self.last_message.get('merkleblock') if not last_filtered_block: return False - # TODO change this method to take a hash value and only return true if the correct block has been received - return True + return last_filtered_block.merkleblock.header.rehash() == int(blockhash, 16) wait_until(test_function, timeout=timeout, lock=mininode_lock) - def wait_for_getdata(self, timeout=60): + def wait_for_getdata(self, hash_list, timeout=60): """Waits for a getdata message. - Receiving any getdata message will satisfy the predicate. the last_message["getdata"] - value must be explicitly cleared before calling this method, or this will return - immediately with success. TODO: change this method to take a hash value and only - return true if the correct block/tx has been requested.""" + The object hashes in the inventory vector must match the provided hash_list.""" def test_function(): assert self.is_connected - return self.last_message.get("getdata") + last_data = self.last_message.get("getdata") + if not last_data: + return False + return [x.hash for x in last_data.inv] == hash_list wait_until(test_function, timeout=timeout, lock=mininode_lock) diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 92725dfcf4..e475ed8596 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Functionality to build scripts, as well as signature hash functions. @@ -97,7 +97,7 @@ class CScriptOp(int): return _opcode_instances[n] except IndexError: assert len(_opcode_instances) == n - _opcode_instances.append(super(CScriptOp, cls).__new__(cls, n)) + _opcode_instances.append(super().__new__(cls, n)) return _opcode_instances[n] # Populate opcode instance table @@ -372,7 +372,7 @@ class CScriptTruncatedPushDataError(CScriptInvalidError): """Invalid pushdata due to truncation""" def __init__(self, msg, data): self.data = data - super(CScriptTruncatedPushDataError, self).__init__(msg) + super().__init__(msg) # This is used, eg, for blockchain heights in coinbase scripts (bip34) @@ -449,15 +449,8 @@ class CScript(bytes): return other def __add__(self, other): - # Do the coercion outside of the try block so that errors in it are - # noticed. - other = self.__coerce_instance(other) - - try: - # bytes.__add__ always returns bytes instances unfortunately - return CScript(super(CScript, self).__add__(other)) - except TypeError: - raise TypeError('Can not add a %r instance to a CScript' % other.__class__) + # add makes no sense for a CScript() + raise NotImplementedError def join(self, iterable): # join makes no sense for a CScript() @@ -465,14 +458,14 @@ class CScript(bytes): def __new__(cls, value=b''): if isinstance(value, bytes) or isinstance(value, bytearray): - return super(CScript, cls).__new__(cls, value) + return super().__new__(cls, value) else: def coerce_iterable(iterable): for instance in iterable: yield cls.__coerce_instance(instance) # Annoyingly on both python2 and python3 bytes.join() always # returns a bytes instance even when subclassed. - return super(CScript, cls).__new__(cls, b''.join(coerce_iterable(value))) + return super().__new__(cls, b''.join(coerce_iterable(value))) def raw_iter(self): """Raw iteration diff --git a/test/functional/test_framework/script_util.py b/test/functional/test_framework/script_util.py index 5ef67226c4..80fbae70bf 100755 --- a/test/functional/test_framework/script_util.py +++ b/test/functional/test_framework/script_util.py @@ -23,3 +23,4 @@ from test_framework.script import CScript # scriptPubKeys are needed, to guarantee that the minimum transaction size is # met. DUMMY_P2WPKH_SCRIPT = CScript([b'a' * 21]) +DUMMY_2_P2WPKH_SCRIPT = CScript([b'b' * 21]) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index e36fb350c6..8719bd0d39 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -165,6 +165,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="run nodes under the valgrind memory error detector: expect at least a ~10x slowdown, valgrind 3.14 or later required") parser.add_argument("--randomseed", type=int, help="set a random seed for deterministically reproducing a previous test run") + parser.add_argument("--descriptors", default=False, action="store_true", + help="Run test using a descriptor wallet") self.add_options(parser) self.options = parser.parse_args() @@ -333,11 +335,23 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def setup_nodes(self): """Override this method to customize test node setup""" - extra_args = None + extra_args = [[]] * self.num_nodes + wallets = [[]] * self.num_nodes if hasattr(self, "extra_args"): extra_args = self.extra_args + wallets = [[x for x in eargs if x.startswith('-wallet=')] for eargs in extra_args] + extra_args = [x + ['-nowallet'] for x in extra_args] self.add_nodes(self.num_nodes, extra_args) self.start_nodes() + for i, n in enumerate(self.nodes): + n.extra_args.pop() + if '-wallet=0' in n.extra_args or '-nowallet' in n.extra_args or '-disablewallet' in n.extra_args or not self.is_wallet_compiled(): + continue + if '-wallet=' not in wallets[i] and not any([x.startswith('-wallet=') for x in wallets[i]]): + wallets[i].append('-wallet=') + for w in wallets[i]: + wallet_name = w.split('=', 1)[1] + n.createwallet(wallet_name=wallet_name, descriptors=self.options.descriptors) self.import_deterministic_coinbase_privkeys() if not self.setup_clean_chain: for n in self.nodes: @@ -408,6 +422,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): use_cli=self.options.usecli, start_perf=self.options.perf, use_valgrind=self.options.valgrind, + descriptors=self.options.descriptors, )) def start_node(self, i, *args, **kwargs): @@ -547,11 +562,16 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): bitcoin_cli=self.options.bitcoincli, coverage_dir=None, cwd=self.options.tmpdir, + descriptors=self.options.descriptors, )) self.start_node(CACHE_NODE_ID) + cache_node = self.nodes[CACHE_NODE_ID] # Wait for RPC connections to be ready - self.nodes[CACHE_NODE_ID].wait_for_rpc_connection() + cache_node.wait_for_rpc_connection() + + # Set a time in the past, so that blocks don't end up in the future + cache_node.setmocktime(cache_node.getblockheader(cache_node.getbestblockhash())['time']) # Create a 199-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. @@ -560,12 +580,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # This is needed so that we are out of IBD when the test starts, # see the tip age check in IsInitialBlockDownload(). for i in range(8): - self.nodes[CACHE_NODE_ID].generatetoaddress( + cache_node.generatetoaddress( nblocks=25 if i != 7 else 24, address=TestNode.PRIV_KEYS[i % 4].address, ) - assert_equal(self.nodes[CACHE_NODE_ID].getblockchaininfo()["blocks"], 199) + assert_equal(cache_node.getblockchaininfo()["blocks"], 199) # Shut it down, and clean up cache directories: self.stop_nodes() diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 8260c917fe..c0075fd8ec 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Class for bitcoind node under test""" @@ -22,10 +22,12 @@ import shlex import sys from .authproxy import JSONRPCException +from .descriptors import descsum_create from .util import ( MAX_NODES, append_config, delete_cookie_file, + get_auth_cookie, get_rpc_proxy, rpc_url, wait_until, @@ -60,7 +62,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, chain, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None): + def __init__(self, i, datadir, *, chain, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -78,6 +80,7 @@ class TestNode(): self.binary = bitcoind self.coverage_dir = coverage_dir self.cwd = cwd + self.descriptors = descriptors if extra_conf is not None: append_config(datadir, extra_conf) # Most callers will just need to add extra args to the standard list below. @@ -169,10 +172,10 @@ class TestNode(): def __getattr__(self, name): """Dispatches any unrecognised messages to the RPC connection or a CLI instance.""" if self.use_cli: - return getattr(self.cli, name) + return getattr(RPCOverloadWrapper(self.cli, True, self.descriptors), name) else: assert self.rpc_connected and self.rpc is not None, self._node_msg("Error: no RPC connection") - return getattr(self.rpc, name) + return getattr(RPCOverloadWrapper(self.rpc, descriptors=self.descriptors), name) def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, **kwargs): """Start the node.""" @@ -225,19 +228,38 @@ class TestNode(): self.rpc_connected = True self.url = self.rpc.url return - except IOError as e: - if e.errno != errno.ECONNREFUSED: # Port not yet open? - raise # unknown IO error except JSONRPCException as e: # Initialization phase # -28 RPC in warmup # -342 Service unavailable, RPC server started but is shutting down due to error if e.error['code'] != -28 and e.error['code'] != -342: raise # unknown JSON RPC exception - except ValueError as e: # cookie file not found and no rpcuser or rpcassword. bitcoind still starting + except ConnectionResetError: + # This might happen when the RPC server is in warmup, but shut down before the call to getblockcount + # succeeds. Try again to properly raise the FailedToStartError + pass + except OSError as e: + if e.errno != errno.ECONNREFUSED: # Port not yet open? + raise # unknown OS error + except ValueError as e: # cookie file not found and no rpcuser or rpcpassword; bitcoind is still starting if "No RPC credentials" not in str(e): raise time.sleep(1.0 / poll_per_s) - self._raise_assertion_error("Unable to connect to bitcoind") + self._raise_assertion_error("Unable to connect to bitcoind after {}s".format(self.rpc_timeout)) + + def wait_for_cookie_credentials(self): + """Ensures auth cookie credentials can be read, e.g. for testing CLI with -rpcwait before RPC connection is up.""" + self.log.debug("Waiting for cookie credentials") + # Poll at a rate of four times per second. + poll_per_s = 4 + for _ in range(poll_per_s * self.rpc_timeout): + try: + get_auth_cookie(self.datadir, self.chain) + self.log.debug("Cookie credentials successfully retrieved") + return + except ValueError: # cookie file not found and no rpcuser or rpcpassword; bitcoind is still starting + pass # so we continue polling until RPC credentials are retrieved + time.sleep(1.0 / poll_per_s) + self._raise_assertion_error("Unable to retrieve cookie credentials after {}s".format(self.rpc_timeout)) def generate(self, nblocks, maxtries=1000000): self.log.debug("TestNode.generate() dispatches `generate` call to `generatetoaddress`") @@ -245,11 +267,11 @@ class TestNode(): def get_wallet_rpc(self, wallet_name): if self.use_cli: - return self.cli("-rpcwallet={}".format(wallet_name)) + return RPCOverloadWrapper(self.cli("-rpcwallet={}".format(wallet_name)), True, self.descriptors) else: assert self.rpc_connected and self.rpc, self._node_msg("RPC not connected") wallet_path = "wallet/{}".format(urllib.parse.quote(wallet_name)) - return self.rpc / wallet_path + return RPCOverloadWrapper(self.rpc / wallet_path, descriptors=self.descriptors) def stop_node(self, expected_stderr='', wait=0): """Stop the node.""" @@ -523,7 +545,6 @@ def arg_to_cli(arg): class TestNodeCLI(): """Interface to bitcoin-cli for an individual node""" - def __init__(self, binary, datadir): self.options = [] self.binary = binary @@ -561,7 +582,7 @@ class TestNodeCLI(): if command is not None: p_args += [command] p_args += pos_args + named_args - self.log.debug("Running bitcoin-cli command: %s" % command) + self.log.debug("Running bitcoin-cli {}".format(p_args[2:])) process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) cli_stdout, cli_stderr = process.communicate(input=self.input) returncode = process.poll() @@ -576,3 +597,118 @@ class TestNodeCLI(): return json.loads(cli_stdout, parse_float=decimal.Decimal) except json.JSONDecodeError: return cli_stdout.rstrip("\n") + +class RPCOverloadWrapper(): + def __init__(self, rpc, cli=False, descriptors=False): + self.rpc = rpc + self.is_cli = cli + self.descriptors = descriptors + + def __getattr__(self, name): + return getattr(self.rpc, name) + + def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase=None, avoid_reuse=None, descriptors=None): + if self.is_cli: + if disable_private_keys is None: + disable_private_keys = 'null' + if blank is None: + blank = 'null' + if passphrase is None: + passphrase = '' + if avoid_reuse is None: + avoid_reuse = 'null' + if descriptors is None: + descriptors = self.descriptors + return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors) + + def importprivkey(self, privkey, label=None, rescan=None): + wallet_info = self.getwalletinfo() + if self.is_cli: + if label is None: + label = 'null' + if rescan is None: + rescan = 'null' + if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): + return self.__getattr__('importprivkey')(privkey, label, rescan) + desc = descsum_create('combo(' + privkey + ')') + req = [{ + 'desc': desc, + 'timestamp': 0 if rescan else 'now', + 'label': label if label else '' + }] + import_res = self.importdescriptors(req) + if not import_res[0]['success']: + raise JSONRPCException(import_res[0]['error']) + + def addmultisigaddress(self, nrequired, keys, label=None, address_type=None): + wallet_info = self.getwalletinfo() + if self.is_cli: + if label is None: + label = 'null' + if address_type is None: + address_type = 'null' + if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): + return self.__getattr__('addmultisigaddress')(nrequired, keys, label, address_type) + cms = self.createmultisig(nrequired, keys, address_type) + req = [{ + 'desc': cms['descriptor'], + 'timestamp': 0, + 'label': label if label else '' + }] + import_res = self.importdescriptors(req) + if not import_res[0]['success']: + raise JSONRPCException(import_res[0]['error']) + return cms + + def importpubkey(self, pubkey, label=None, rescan=None): + wallet_info = self.getwalletinfo() + if self.is_cli: + if label is None: + label = 'null' + if rescan is None: + rescan = 'null' + if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): + return self.__getattr__('importpubkey')(pubkey, label, rescan) + desc = descsum_create('combo(' + pubkey + ')') + req = [{ + 'desc': desc, + 'timestamp': 0 if rescan else 'now', + 'label': label if label else '' + }] + import_res = self.importdescriptors(req) + if not import_res[0]['success']: + raise JSONRPCException(import_res[0]['error']) + + def importaddress(self, address, label=None, rescan=None, p2sh=None): + wallet_info = self.getwalletinfo() + if self.is_cli: + if label is None: + label = 'null' + if rescan is None: + rescan = 'null' + if p2sh is None: + p2sh = 'null' + if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): + return self.__getattr__('importaddress')(address, label, rescan, p2sh) + is_hex = False + try: + int(address ,16) + is_hex = True + desc = descsum_create('raw(' + address + ')') + except: + desc = descsum_create('addr(' + address + ')') + reqs = [{ + 'desc': desc, + 'timestamp': 0 if rescan else 'now', + 'label': label if label else '' + }] + if is_hex and p2sh: + reqs.append({ + 'desc': descsum_create('p2sh(raw(' + address + '))'), + 'timestamp': 0 if rescan else 'now', + 'label': label if label else '' + }) + import_res = self.importdescriptors(reqs) + for res in import_res: + if not res['success']: + raise JSONRPCException(res['error']) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index e89b4e9879..64e1aa3bbc 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helpful routines for regression testing.""" @@ -410,7 +410,10 @@ def sync_blocks(rpc_connections, *, wait=1, timeout=60): # Check that each peer has at least one connection assert (all([len(x.getpeerinfo()) for x in rpc_connections])) time.sleep(wait) - raise AssertionError("Block sync timed out:{}".format("".join("\n {!r}".format(b) for b in best_hash))) + raise AssertionError("Block sync timed out after {}s:{}".format( + timeout, + "".join("\n {!r}".format(b) for b in best_hash), + )) def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True): @@ -429,11 +432,16 @@ def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True): # Check that each peer has at least one connection assert (all([len(x.getpeerinfo()) for x in rpc_connections])) time.sleep(wait) - raise AssertionError("Mempool sync timed out:{}".format("".join("\n {!r}".format(m) for m in pool))) + raise AssertionError("Mempool sync timed out after {}s:{}".format( + timeout, + "".join("\n {!r}".format(m) for m in pool), + )) + # Transaction/Block functions ############################# + def find_output(node, txid, amount, *, blockhash=None): """ Return index to output of txid with value amount diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index eb537015fb..1b6686ff45 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -13,6 +13,10 @@ from test_framework.address import ( script_to_p2sh_p2wsh, script_to_p2wsh, ) +from test_framework.key import ( + bytes_to_wif, + ECKey, +) from test_framework.script import ( CScript, OP_0, @@ -66,6 +70,25 @@ def get_key(node): p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(), p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey)) +def get_generate_key(): + """Generate a fresh key + + Returns a named tuple of privkey, pubkey and all address and scripts.""" + eckey = ECKey() + eckey.generate() + privkey = bytes_to_wif(eckey.get_bytes()) + pubkey = eckey.get_pubkey().get_bytes().hex() + pkh = hash160(hex_str_to_bytes(pubkey)) + return Key(privkey=privkey, + pubkey=pubkey, + p2pkh_script=CScript([OP_DUP, OP_HASH160, pkh, OP_EQUALVERIFY, OP_CHECKSIG]).hex(), + p2pkh_addr=key_to_p2pkh(pubkey), + p2wpkh_script=CScript([OP_0, pkh]).hex(), + p2wpkh_addr=key_to_p2wpkh(pubkey), + p2sh_p2wpkh_script=CScript([OP_HASH160, hash160(CScript([OP_0, pkh])), OP_EQUAL]).hex(), + p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(), + p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey)) + def get_multisig(node): """Generate a fresh 2-of-3 multisig on node diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 2f307750a9..b8523e16b7 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run regression test suite. @@ -76,6 +76,7 @@ BASE_SCRIPTS = [ # Scripts that are run by default. # Longest test should go first, to favor running tests in parallel 'wallet_hd.py', + 'wallet_hd.py --descriptors', 'wallet_backup.py', # vv Tests less than 5m vv 'mining_getblocktemplate_longpoll.py', @@ -86,7 +87,9 @@ BASE_SCRIPTS = [ 'feature_segwit.py', # vv Tests less than 2m vv 'wallet_basic.py', + 'wallet_basic.py --descriptors', 'wallet_labels.py', + 'wallet_labels.py --descriptors', 'p2p_segwit.py', 'p2p_timeouts.py', 'p2p_tx_download.py', @@ -109,6 +112,7 @@ BASE_SCRIPTS = [ 'feature_abortnode.py', # vv Tests less than 30s vv 'wallet_keypool_topup.py', + 'wallet_keypool_topup.py --descriptors', 'feature_fee_estimation.py', 'interface_zmq.py', 'interface_bitcoin_cli.py', @@ -122,6 +126,7 @@ BASE_SCRIPTS = [ 'interface_rest.py', 'mempool_spend_coinbase.py', 'wallet_avoidreuse.py', + 'wallet_avoidreuse.py --descriptors', 'mempool_reorg.py', 'mempool_persist.py', 'wallet_multiwallet.py', @@ -134,6 +139,7 @@ BASE_SCRIPTS = [ 'interface_http.py', 'interface_rpc.py', 'rpc_psbt.py', + 'rpc_psbt.py --descriptors', 'rpc_users.py', 'rpc_whitelist.py', 'feature_proxy.py', @@ -144,8 +150,11 @@ BASE_SCRIPTS = [ 'rpc_blockchain.py', 'rpc_deprecated.py', 'wallet_disable.py', + 'p2p_addr_relay.py', 'rpc_net.py', 'wallet_keypool.py', + 'wallet_keypool.py --descriptors', + 'wallet_descriptor.py', 'p2p_mempool.py', 'p2p_filter.py', 'rpc_setban.py', @@ -167,17 +176,20 @@ BASE_SCRIPTS = [ 'mempool_packages.py', 'mempool_package_onemore.py', 'rpc_createmultisig.py', + 'rpc_createmultisig.py --descriptors', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', 'wallet_importprunedfunds.py', 'p2p_leak_tx.py', 'rpc_signmessage.py', + 'rpc_generateblock.py', 'wallet_balance.py', 'feature_nulldummy.py', 'mempool_accept.py', 'mempool_expiry.py', 'wallet_import_rescan.py', 'wallet_import_with_label.py', + 'wallet_importdescriptors.py', 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', @@ -188,6 +200,7 @@ BASE_SCRIPTS = [ 'wallet_listsinceblock.py', 'p2p_leak.py', 'wallet_encryption.py', + 'wallet_encryption.py --descriptors', 'feature_dersig.py', 'feature_cltv.py', 'rpc_uptime.py', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index d2629ff1ed..b3d496dd51 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-wallet.""" diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 1122daaf83..c7b19081c8 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the abandontransaction RPC. @@ -110,8 +110,8 @@ class AbandonConflictTest(BitcoinTestFramework): assert_equal(newbalance, balance - signed3_change) # Unconfirmed received funds that are not in mempool, also shouldn't show # up in unconfirmed balance - unconfbalance = self.nodes[0].getunconfirmedbalance() + self.nodes[0].getbalance() - assert_equal(unconfbalance, newbalance) + balances = self.nodes[0].getbalances()['mine'] + assert_equal(balances['untrusted_pending'] + balances['trusted'], newbalance) # Also shouldn't show up in listunspent assert not txABC2 in [utxo["txid"] for utxo in self.nodes[0].listunspent(0)] balance = newbalance diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index 79b6db986b..68e22b7e86 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet can send and receive using all combinations of address types. @@ -97,12 +97,9 @@ class AddressTypeTest(BitcoinTestFramework): connect_nodes(self.nodes[i], j) self.sync_all() - def get_balances(self, confirmed=True): - """Return a list of confirmed or unconfirmed balances.""" - if confirmed: - return [self.nodes[i].getbalance() for i in range(4)] - else: - return [self.nodes[i].getunconfirmedbalance() for i in range(4)] + def get_balances(self, key='trusted'): + """Return a list of balances.""" + return [self.nodes[i].getbalances()['mine'][key] for i in range(4)] # Quick test of python bech32 implementation def test_python_bech32(self, addr): @@ -307,7 +304,7 @@ class AddressTypeTest(BitcoinTestFramework): self.nodes[from_node].sendmany("", sends) self.sync_mempools() - unconf_balances = self.get_balances(False) + unconf_balances = self.get_balances('untrusted_pending') self.log.debug("Check unconfirmed balances: {}".format(unconf_balances)) assert_equal(unconf_balances[from_node], 0) for n, to_node in enumerate(range(from_node + 1, from_node + 4)): diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 2ce8d459c6..780cce9d02 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -85,15 +85,19 @@ class AvoidReuseTest(BitcoinTestFramework): self.sync_all() self.test_change_remains_change(self.nodes[1]) reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) - self.test_fund_send_fund_senddirty() + self.test_sending_from_reused_address_without_avoid_reuse() reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) - self.test_fund_send_fund_send("legacy") + self.test_sending_from_reused_address_fails("legacy") reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) - self.test_fund_send_fund_send("p2sh-segwit") + self.test_sending_from_reused_address_fails("p2sh-segwit") reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) - self.test_fund_send_fund_send("bech32") + self.test_sending_from_reused_address_fails("bech32") reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) self.test_getbalances_used() + reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) + self.test_full_destination_group_is_preferred() + reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) + self.test_all_destination_groups_are_used() def test_persistence(self): '''Test that wallet files persist the avoid_reuse flag.''' @@ -129,7 +133,7 @@ class AvoidReuseTest(BitcoinTestFramework): tempwallet = ".wallet_avoidreuse.py_test_immutable_wallet.dat" # Create a wallet with disable_private_keys set; this should work - self.nodes[1].createwallet(tempwallet, True) + self.nodes[1].createwallet(wallet_name=tempwallet, disable_private_keys=True) w = self.nodes[1].get_wallet_rpc(tempwallet) # Attempt to unset the disable_private_keys flag; this should not work @@ -162,13 +166,13 @@ class AvoidReuseTest(BitcoinTestFramework): for logical_tx in node.listtransactions(): assert logical_tx.get('address') != changeaddr - def test_fund_send_fund_senddirty(self): + def test_sending_from_reused_address_without_avoid_reuse(self): ''' - Test the same as test_fund_send_fund_send, except send the 10 BTC with + Test the same as test_sending_from_reused_address_fails, except send the 10 BTC with the avoid_reuse flag set to false. This means the 10 BTC send should succeed, - where it fails in test_fund_send_fund_send. + where it fails in test_sending_from_reused_address_fails. ''' - self.log.info("Test fund send fund send dirty") + self.log.info("Test sending from reused address with avoid_reuse=false") fundaddr = self.nodes[1].getnewaddress() retaddr = self.nodes[0].getnewaddress() @@ -213,7 +217,7 @@ class AvoidReuseTest(BitcoinTestFramework): assert_approx(self.nodes[1].getbalance(), 5, 0.001) assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 5, 0.001) - def test_fund_send_fund_send(self, second_addr_type): + def test_sending_from_reused_address_fails(self, second_addr_type): ''' Test the simple case where [1] generates a new address A, then [0] sends 10 BTC to A. @@ -222,7 +226,7 @@ class AvoidReuseTest(BitcoinTestFramework): [1] tries to spend 10 BTC (fails; dirty). [1] tries to spend 4 BTC (succeeds; change address sufficient) ''' - self.log.info("Test fund send fund send") + self.log.info("Test sending from reused {} address fails".format(second_addr_type)) fundaddr = self.nodes[1].getnewaddress(label="", address_type="legacy") retaddr = self.nodes[0].getnewaddress() @@ -245,43 +249,44 @@ class AvoidReuseTest(BitcoinTestFramework): # getbalances should show no used, 5 btc trusted assert_balances(self.nodes[1], mine={"used": 0, "trusted": 5}) - # For the second send, we transmute it to a related single-key address - # to make sure it's also detected as re-use - fund_spk = self.nodes[0].getaddressinfo(fundaddr)["scriptPubKey"] - fund_decoded = self.nodes[0].decodescript(fund_spk) - if second_addr_type == "p2sh-segwit": - new_fundaddr = fund_decoded["segwit"]["p2sh-segwit"] - elif second_addr_type == "bech32": - new_fundaddr = fund_decoded["segwit"]["addresses"][0] - else: - new_fundaddr = fundaddr - assert_equal(second_addr_type, "legacy") - - self.nodes[0].sendtoaddress(new_fundaddr, 10) - self.nodes[0].generate(1) - self.sync_all() - - # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10) - assert_unspent(self.nodes[1], total_count=2, total_sum=15, reused_count=1, reused_sum=10) - # getbalances should show 10 used, 5 btc trusted - assert_balances(self.nodes[1], mine={"used": 10, "trusted": 5}) - - # node 1 should now have a balance of 5 (no dirty) or 15 (including dirty) - assert_approx(self.nodes[1].getbalance(), 5, 0.001) - assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 15, 0.001) - - assert_raises_rpc_error(-6, "Insufficient funds", self.nodes[1].sendtoaddress, retaddr, 10) - - self.nodes[1].sendtoaddress(retaddr, 4) - - # listunspent should show 2 total outputs (1, 10 btc), one unused (1), one reused (10) - assert_unspent(self.nodes[1], total_count=2, total_sum=11, reused_count=1, reused_sum=10) - # getbalances should show 10 used, 1 btc trusted - assert_balances(self.nodes[1], mine={"used": 10, "trusted": 1}) - - # node 1 should now have about 1 btc left (no dirty) and 11 (including dirty) - assert_approx(self.nodes[1].getbalance(), 1, 0.001) - assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 11, 0.001) + if not self.options.descriptors: + # For the second send, we transmute it to a related single-key address + # to make sure it's also detected as re-use + fund_spk = self.nodes[0].getaddressinfo(fundaddr)["scriptPubKey"] + fund_decoded = self.nodes[0].decodescript(fund_spk) + if second_addr_type == "p2sh-segwit": + new_fundaddr = fund_decoded["segwit"]["p2sh-segwit"] + elif second_addr_type == "bech32": + new_fundaddr = fund_decoded["segwit"]["addresses"][0] + else: + new_fundaddr = fundaddr + assert_equal(second_addr_type, "legacy") + + self.nodes[0].sendtoaddress(new_fundaddr, 10) + self.nodes[0].generate(1) + self.sync_all() + + # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10) + assert_unspent(self.nodes[1], total_count=2, total_sum=15, reused_count=1, reused_sum=10) + # getbalances should show 10 used, 5 btc trusted + assert_balances(self.nodes[1], mine={"used": 10, "trusted": 5}) + + # node 1 should now have a balance of 5 (no dirty) or 15 (including dirty) + assert_approx(self.nodes[1].getbalance(), 5, 0.001) + assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 15, 0.001) + + assert_raises_rpc_error(-6, "Insufficient funds", self.nodes[1].sendtoaddress, retaddr, 10) + + self.nodes[1].sendtoaddress(retaddr, 4) + + # listunspent should show 2 total outputs (1, 10 btc), one unused (1), one reused (10) + assert_unspent(self.nodes[1], total_count=2, total_sum=11, reused_count=1, reused_sum=10) + # getbalances should show 10 used, 1 btc trusted + assert_balances(self.nodes[1], mine={"used": 10, "trusted": 1}) + + # node 1 should now have about 1 btc left (no dirty) and 11 (including dirty) + assert_approx(self.nodes[1].getbalance(), 1, 0.001) + assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 11, 0.001) def test_getbalances_used(self): ''' @@ -313,5 +318,66 @@ class AvoidReuseTest(BitcoinTestFramework): assert_unspent(self.nodes[1], total_count=2, total_sum=6, reused_count=1, reused_sum=1) assert_balances(self.nodes[1], mine={"used": 1, "trusted": 5}) + def test_full_destination_group_is_preferred(self): + ''' + Test the case where [1] only has 11 outputs of 1 BTC in the same reused + address and tries to send a small payment of 0.5 BTC. The wallet + should use 10 outputs from the reused address as inputs and not a + single 1 BTC input, in order to join several outputs from the reused + address. + ''' + self.log.info("Test that full destination groups are preferred in coin selection") + + # Node under test should be empty + assert_equal(self.nodes[1].getbalance(avoid_reuse=False), 0) + + new_addr = self.nodes[1].getnewaddress() + ret_addr = self.nodes[0].getnewaddress() + + # Send 11 outputs of 1 BTC to the same, reused address in the wallet + for _ in range(11): + self.nodes[0].sendtoaddress(new_addr, 1) + + self.nodes[0].generate(1) + self.sync_all() + + # Sending a transaction that is smaller than each one of the + # available outputs + txid = self.nodes[1].sendtoaddress(address=ret_addr, amount=0.5) + inputs = self.nodes[1].getrawtransaction(txid, 1)["vin"] + + # The transaction should use 10 inputs exactly + assert_equal(len(inputs), 10) + + def test_all_destination_groups_are_used(self): + ''' + Test the case where [1] only has 22 outputs of 1 BTC in the same reused + address and tries to send a payment of 20.5 BTC. The wallet + should use all 22 outputs from the reused address as inputs. + ''' + self.log.info("Test that all destination groups are used") + + # Node under test should be empty + assert_equal(self.nodes[1].getbalance(avoid_reuse=False), 0) + + new_addr = self.nodes[1].getnewaddress() + ret_addr = self.nodes[0].getnewaddress() + + # Send 22 outputs of 1 BTC to the same, reused address in the wallet + for _ in range(22): + self.nodes[0].sendtoaddress(new_addr, 1) + + self.nodes[0].generate(1) + self.sync_all() + + # Sending a transaction that needs to use the full groups + # of 10 inputs but also the incomplete group of 2 inputs. + txid = self.nodes[1].sendtoaddress(address=ret_addr, amount=20.5) + inputs = self.nodes[1].getrawtransaction(txid, 1)["vin"] + + # The transaction should use 22 inputs exactly + assert_equal(len(inputs), 22) + + if __name__ == '__main__': AvoidReuseTest().main() diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index fb80a06433..9dd91b2495 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet backup features. diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index a5f9a047ed..8efa66a856 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet balance RPC methods.""" @@ -107,7 +107,7 @@ class WalletTest(BitcoinTestFramework): # First argument of getbalance must be set to "*" assert_raises_rpc_error(-32, "dummy first argument must be excluded or set to \"*\"", self.nodes[1].getbalance, "") - self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs") + self.log.info("Test balances with unconfirmed inputs") # Before `test_balance()`, we have had two nodes with a balance of 50 # each and then we: @@ -148,6 +148,18 @@ class WalletTest(BitcoinTestFramework): def test_balances(*, fee_node_1=0): + # getbalances + expected_balances_0 = {'mine': {'immature': Decimal('0E-8'), + 'trusted': Decimal('9.99'), # change from node 0's send + 'untrusted_pending': Decimal('60.0')}, + 'watchonly': {'immature': Decimal('5000'), + 'trusted': Decimal('50.0'), + 'untrusted_pending': Decimal('0E-8')}} + expected_balances_1 = {'mine': {'immature': Decimal('0E-8'), + 'trusted': Decimal('0E-8'), # node 1's send had an unsafe input + 'untrusted_pending': Decimal('30.0') - fee_node_1}} # Doesn't include output of node 0's send since it was spent + assert_equal(self.nodes[0].getbalances(), expected_balances_0) + assert_equal(self.nodes[1].getbalances(), expected_balances_1) # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send assert_equal(self.nodes[1].getbalance(), Decimal('0')) # node 1's send had an unsafe input @@ -160,11 +172,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0')) # getunconfirmedbalance assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) # output of node 1's spend - assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('60')) - assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) - assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('30') - fee_node_1) # Doesn't include output of node 0's send since it was spent - assert_equal(self.nodes[1].getbalances()['mine']['untrusted_pending'], Decimal('30') - fee_node_1) + # getwalletinfo.unconfirmed_balance + assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('30') - fee_node_1) test_balances(fee_node_1=Decimal('0.01')) @@ -174,15 +184,19 @@ class WalletTest(BitcoinTestFramework): self.nodes[0].sendrawtransaction(txs[1]['hex']) # sending on both nodes is faster than waiting for propagation self.sync_all() - self.log.info("Test getbalance and getunconfirmedbalance with conflicted unconfirmed inputs") + self.log.info("Test getbalance and getbalances.mine.untrusted_pending with conflicted unconfirmed inputs") test_balances(fee_node_1=Decimal('0.02')) self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY) self.sync_all() # balances are correct after the transactions are confirmed - assert_equal(self.nodes[0].getbalance(), Decimal('69.99')) # node 1's send plus change from node 0's send - assert_equal(self.nodes[1].getbalance(), Decimal('29.98')) # change from node 0's send + balance_node0 = Decimal('69.99') # node 1's send plus change from node 0's send + balance_node1 = Decimal('29.98') # change from node 0's send + assert_equal(self.nodes[0].getbalances()['mine']['trusted'], balance_node0) + assert_equal(self.nodes[1].getbalances()['mine']['trusted'], balance_node1) + assert_equal(self.nodes[0].getbalance(), balance_node0) + assert_equal(self.nodes[1].getbalance(), balance_node1) # Send total balance away from node 1 txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')]) @@ -200,13 +214,13 @@ class WalletTest(BitcoinTestFramework): # check mempool transactions count for wallet unconfirmed balance after # dynamically loading the wallet. - before = self.nodes[1].getunconfirmedbalance() + before = self.nodes[1].getbalances()['mine']['untrusted_pending'] dst = self.nodes[1].getnewaddress() self.nodes[1].unloadwallet('') self.nodes[0].sendtoaddress(dst, 0.1) self.sync_all() self.nodes[1].loadwallet('') - after = self.nodes[1].getunconfirmedbalance() + after = self.nodes[1].getbalances()['mine']['untrusted_pending'] assert_equal(before + Decimal('0.1'), after) # Create 3 more wallet txs, where the last is not accepted to the diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 15746d312c..2dddbf2cf3 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -49,6 +49,7 @@ class WalletTest(BitcoinTestFramework): return self.nodes[0].decoderawtransaction(txn)['vsize'] def run_test(self): + # Check that there's no UTXO on none of the nodes assert_equal(len(self.nodes[0].listunspent()), 0) assert_equal(len(self.nodes[1].listunspent()), 0) @@ -219,7 +220,7 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), node_2_bal) node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) - self.start_node(3) + self.start_node(3, self.nodes[3].extra_args) connect_nodes(self.nodes[0], 3) self.sync_all() @@ -315,57 +316,59 @@ class WalletTest(BitcoinTestFramework): # This will raise an exception since generate does not accept a string assert_raises_rpc_error(-1, "not an integer", self.nodes[0].generate, "2") - # This will raise an exception for the invalid private key format - assert_raises_rpc_error(-5, "Invalid private key encoding", self.nodes[0].importprivkey, "invalid") + if not self.options.descriptors: - # This will raise an exception for importing an address with the PS2H flag - temp_address = self.nodes[1].getnewaddress("", "p2sh-segwit") - assert_raises_rpc_error(-5, "Cannot use the p2sh flag with an address - use a script instead", self.nodes[0].importaddress, temp_address, "label", False, True) + # This will raise an exception for the invalid private key format + assert_raises_rpc_error(-5, "Invalid private key encoding", self.nodes[0].importprivkey, "invalid") - # This will raise an exception for attempting to dump the private key of an address you do not own - assert_raises_rpc_error(-3, "Address does not refer to a key", self.nodes[0].dumpprivkey, temp_address) + # This will raise an exception for importing an address with the PS2H flag + temp_address = self.nodes[1].getnewaddress("", "p2sh-segwit") + assert_raises_rpc_error(-5, "Cannot use the p2sh flag with an address - use a script instead", self.nodes[0].importaddress, temp_address, "label", False, True) - # This will raise an exception for attempting to get the private key of an invalid Bitcoin address - assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].dumpprivkey, "invalid") + # This will raise an exception for attempting to dump the private key of an address you do not own + assert_raises_rpc_error(-3, "Address does not refer to a key", self.nodes[0].dumpprivkey, temp_address) - # This will raise an exception for attempting to set a label for an invalid Bitcoin address - assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].setlabel, "invalid address", "label") + # This will raise an exception for attempting to get the private key of an invalid Bitcoin address + assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].dumpprivkey, "invalid") - # This will raise an exception for importing an invalid address - assert_raises_rpc_error(-5, "Invalid Bitcoin address or script", self.nodes[0].importaddress, "invalid") + # This will raise an exception for attempting to set a label for an invalid Bitcoin address + assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].setlabel, "invalid address", "label") - # This will raise an exception for attempting to import a pubkey that isn't in hex - assert_raises_rpc_error(-5, "Pubkey must be a hex string", self.nodes[0].importpubkey, "not hex") + # This will raise an exception for importing an invalid address + assert_raises_rpc_error(-5, "Invalid Bitcoin address or script", self.nodes[0].importaddress, "invalid") - # This will raise an exception for importing an invalid pubkey - assert_raises_rpc_error(-5, "Pubkey is not a valid public key", self.nodes[0].importpubkey, "5361746f736869204e616b616d6f746f") + # This will raise an exception for attempting to import a pubkey that isn't in hex + assert_raises_rpc_error(-5, "Pubkey must be a hex string", self.nodes[0].importpubkey, "not hex") - # Import address and private key to check correct behavior of spendable unspents - # 1. Send some coins to generate new UTXO - address_to_import = self.nodes[2].getnewaddress() - txid = self.nodes[0].sendtoaddress(address_to_import, 1) - self.nodes[0].generate(1) - self.sync_all(self.nodes[0:3]) + # This will raise an exception for importing an invalid pubkey + assert_raises_rpc_error(-5, "Pubkey is not a valid public key", self.nodes[0].importpubkey, "5361746f736869204e616b616d6f746f") + + # Import address and private key to check correct behavior of spendable unspents + # 1. Send some coins to generate new UTXO + address_to_import = self.nodes[2].getnewaddress() + txid = self.nodes[0].sendtoaddress(address_to_import, 1) + self.nodes[0].generate(1) + self.sync_all(self.nodes[0:3]) - # 2. Import address from node2 to node1 - self.nodes[1].importaddress(address_to_import) + # 2. Import address from node2 to node1 + self.nodes[1].importaddress(address_to_import) - # 3. Validate that the imported address is watch-only on node1 - assert self.nodes[1].getaddressinfo(address_to_import)["iswatchonly"] + # 3. Validate that the imported address is watch-only on node1 + assert self.nodes[1].getaddressinfo(address_to_import)["iswatchonly"] - # 4. Check that the unspents after import are not spendable - assert_array_result(self.nodes[1].listunspent(), - {"address": address_to_import}, - {"spendable": False}) + # 4. Check that the unspents after import are not spendable + assert_array_result(self.nodes[1].listunspent(), + {"address": address_to_import}, + {"spendable": False}) - # 5. Import private key of the previously imported address on node1 - priv_key = self.nodes[2].dumpprivkey(address_to_import) - self.nodes[1].importprivkey(priv_key) + # 5. Import private key of the previously imported address on node1 + priv_key = self.nodes[2].dumpprivkey(address_to_import) + self.nodes[1].importprivkey(priv_key) - # 6. Check that the unspents are now spendable on node1 - assert_array_result(self.nodes[1].listunspent(), - {"address": address_to_import}, - {"spendable": True}) + # 6. Check that the unspents are now spendable on node1 + assert_array_result(self.nodes[1].listunspent(), + {"address": address_to_import}, + {"spendable": True}) # Mine a block from node0 to an address from node1 coinbase_addr = self.nodes[1].getnewaddress() @@ -460,7 +463,8 @@ class WalletTest(BitcoinTestFramework): # Try with walletrejectlongchains # Double chain limit but require combining inputs, so we pass SelectCoinsMinConf self.stop_node(0) - self.start_node(0, extra_args=["-walletrejectlongchains", "-limitancestorcount=" + str(2 * chainlimit)]) + extra_args = ["-walletrejectlongchains", "-limitancestorcount=" + str(2 * chainlimit)] + self.start_node(0, extra_args=extra_args) # wait for loadmempool timeout = 10 diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 0b3dea94d5..c09ca8854f 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the bumpfee RPC. @@ -82,7 +82,6 @@ class BumpFeeTest(BitcoinTestFramework): test_notmine_bumpfee_fails(self, rbf_node, peer_node, dest_address) test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_address) test_dust_to_fee(self, rbf_node, dest_address) - test_settxfee(self, rbf_node, dest_address) test_watchonly_psbt(self, peer_node, rbf_node, dest_address) test_rebumping(self, rbf_node, dest_address) test_rebumping_not_replaceable(self, rbf_node, dest_address) @@ -90,6 +89,7 @@ class BumpFeeTest(BitcoinTestFramework): test_bumpfee_metadata(self, rbf_node, dest_address) test_locked_wallet_fails(self, rbf_node, dest_address) test_change_script_match(self, rbf_node, dest_address) + test_settxfee(self, rbf_node, dest_address) test_maxtxfee_fails(self, rbf_node, dest_address) # These tests wipe out a number of utxos that are expected in other tests test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) @@ -286,9 +286,15 @@ def test_settxfee(self, rbf_node, dest_address): assert_greater_than(Decimal("0.00001000"), abs(requested_feerate - actual_feerate)) rbf_node.settxfee(Decimal("0.00000000")) # unset paytxfee + # check that settxfee respects -maxtxfee + self.restart_node(1, ['-maxtxfee=0.000025'] + self.extra_args[1]) + assert_raises_rpc_error(-8, "txfee cannot be more than wallet max tx fee", rbf_node.settxfee, Decimal('0.00003')) + self.restart_node(1, self.extra_args[1]) + rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + def test_maxtxfee_fails(self, rbf_node, dest_address): - self.log.info('Test that bumpfee fails when it hits -matxfee') + self.log.info('Test that bumpfee fails when it hits -maxtxfee') # size of bumped transaction (p2wpkh, 1 input, 2 outputs): 141 vbytes # expected bump fee of 141 vbytes * 0.00200000 BTC / 1000 vbytes = 0.00002820 BTC # which exceeds maxtxfee and is expected to raise diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index b24d312e27..48c0fcb731 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test createwallet arguments. diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py new file mode 100755 index 0000000000..289ccf43ec --- /dev/null +++ b/test/functional/wallet_descriptor.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test descriptor wallet function.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error +) + + +class WalletDescriptorTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [['-keypool=100']] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + # Make a descriptor wallet + self.log.info("Making a descriptor wallet") + self.nodes[0].createwallet(wallet_name="desc1", descriptors=True) + self.nodes[0].unloadwallet("") + + # A descriptor wallet should have 100 addresses * 3 types = 300 keys + self.log.info("Checking wallet info") + wallet_info = self.nodes[0].getwalletinfo() + assert_equal(wallet_info['keypoolsize'], 300) + assert_equal(wallet_info['keypoolsize_hd_internal'], 300) + assert 'keypoololdest' not in wallet_info + + # Check that getnewaddress works + self.log.info("Test that getnewaddress and getrawchangeaddress work") + addr = self.nodes[0].getnewaddress("", "legacy") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('pkh(') + assert_equal(addr_info['hdkeypath'], 'm/44\'/1\'/0\'/0/0') + + addr = self.nodes[0].getnewaddress("", "p2sh-segwit") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('sh(wpkh(') + assert_equal(addr_info['hdkeypath'], 'm/49\'/1\'/0\'/0/0') + + addr = self.nodes[0].getnewaddress("", "bech32") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('wpkh(') + assert_equal(addr_info['hdkeypath'], 'm/84\'/1\'/0\'/0/0') + + # Check that getrawchangeaddress works + addr = self.nodes[0].getrawchangeaddress("legacy") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('pkh(') + assert_equal(addr_info['hdkeypath'], 'm/44\'/1\'/0\'/1/0') + + addr = self.nodes[0].getrawchangeaddress("p2sh-segwit") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('sh(wpkh(') + assert_equal(addr_info['hdkeypath'], 'm/49\'/1\'/0\'/1/0') + + addr = self.nodes[0].getrawchangeaddress("bech32") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('wpkh(') + assert_equal(addr_info['hdkeypath'], 'm/84\'/1\'/0\'/1/0') + + # Make a wallet to receive coins at + self.nodes[0].createwallet(wallet_name="desc2", descriptors=True) + recv_wrpc = self.nodes[0].get_wallet_rpc("desc2") + send_wrpc = self.nodes[0].get_wallet_rpc("desc1") + + # Generate some coins + send_wrpc.generatetoaddress(101, send_wrpc.getnewaddress()) + + # Make transactions + self.log.info("Test sending and receiving") + addr = recv_wrpc.getnewaddress() + send_wrpc.sendtoaddress(addr, 10) + + # Make sure things are disabled + self.log.info("Test disabled RPCs") + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.importprivkey, "cVpF924EspNh8KjYsfhgY96mmxvT6DgdWiTYMtMjuM74hJaU5psW") + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.importpubkey, send_wrpc.getaddressinfo(send_wrpc.getnewaddress())) + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.importaddress, recv_wrpc.getnewaddress()) + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.importmulti, []) + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.addmultisigaddress, 1, [recv_wrpc.getnewaddress()]) + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.dumpprivkey, recv_wrpc.getnewaddress()) + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.dumpwallet, 'wallet.dump') + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.importwallet, 'wallet.dump') + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.sethdseed) + + self.log.info("Test encryption") + # Get the master fingerprint before encrypt + info1 = send_wrpc.getaddressinfo(send_wrpc.getnewaddress()) + + # Encrypt wallet 0 + send_wrpc.encryptwallet('pass') + send_wrpc.walletpassphrase('pass', 10) + addr = send_wrpc.getnewaddress() + info2 = send_wrpc.getaddressinfo(addr) + assert info1['hdmasterfingerprint'] != info2['hdmasterfingerprint'] + send_wrpc.walletlock() + assert 'hdmasterfingerprint' in send_wrpc.getaddressinfo(send_wrpc.getnewaddress()) + info3 = send_wrpc.getaddressinfo(addr) + assert_equal(info2['desc'], info3['desc']) + + self.log.info("Test that getnewaddress still works after keypool is exhausted in an encrypted wallet") + for i in range(0, 500): + send_wrpc.getnewaddress() + + self.log.info("Test that unlock is needed when deriving only hardened keys in an encrypted wallet") + send_wrpc.walletpassphrase('pass', 10) + send_wrpc.importdescriptors([{ + "desc": "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/*h)#y4dfsj7n", + "timestamp": "now", + "range": [0,10], + "active": True + }]) + send_wrpc.walletlock() + # Exhaust keypool of 100 + for i in range(0, 100): + send_wrpc.getnewaddress(address_type='bech32') + # This should now error + assert_raises_rpc_error(-12, "Keypool ran out, please call keypoolrefill first", send_wrpc.getnewaddress, '', 'bech32') + + self.log.info("Test born encrypted wallets") + self.nodes[0].createwallet('desc_enc', False, False, 'pass', False, True) + enc_rpc = self.nodes[0].get_wallet_rpc('desc_enc') + enc_rpc.getnewaddress() # Makes sure that we can get a new address from a born encrypted wallet + + self.log.info("Test blank descriptor wallets") + self.nodes[0].createwallet(wallet_name='desc_blank', blank=True, descriptors=True) + blank_rpc = self.nodes[0].get_wallet_rpc('desc_blank') + assert_raises_rpc_error(-4, 'This wallet has no available keys', blank_rpc.getnewaddress) + + self.log.info("Test descriptor wallet with disabled private keys") + self.nodes[0].createwallet(wallet_name='desc_no_priv', disable_private_keys=True, descriptors=True) + nopriv_rpc = self.nodes[0].get_wallet_rpc('desc_no_priv') + assert_raises_rpc_error(-4, 'This wallet has no available keys', nopriv_rpc.getnewaddress) + +if __name__ == '__main__': + WalletDescriptorTest().main () diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index a39dfc7895..cc349c7bc5 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" +import datetime import os +import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -18,6 +20,7 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): Also check that the old hd_master is inactive """ with open(file_name, encoding='utf8') as inputfile: + found_comments = [] found_legacy_addr = 0 found_p2sh_segwit_addr = 0 found_bech32_addr = 0 @@ -26,8 +29,12 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): found_addr_rsv = 0 hd_master_addr_ret = None for line in inputfile: - # only read non comment lines - if line[0] != "#" and len(line) > 10: + line = line.strip() + if not line: + continue + if line[0] == '#': + found_comments.append(line) + else: # split out some data key_date_label, comment = line.split("#") key_date_label = key_date_label.split(" ") @@ -82,7 +89,7 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): found_script_addr += 1 break - return found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret + return found_comments, found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret class WalletDumpTest(BitcoinTestFramework): @@ -122,12 +129,36 @@ class WalletDumpTest(BitcoinTestFramework): # its capacity self.nodes[0].keypoolrefill() - # dump unencrypted wallet + self.log.info('Mine a block one second before the wallet is dumped') + dump_time = int(time.time()) + self.nodes[0].setmocktime(dump_time - 1) + self.nodes[0].generate(1) + self.nodes[0].setmocktime(dump_time) + dump_time_str = '# * Created on {}Z'.format( + datetime.datetime.fromtimestamp( + dump_time, + tz=datetime.timezone.utc, + ).replace(tzinfo=None).isoformat()) + dump_best_block_1 = '# * Best block at time of backup was {} ({}),'.format( + self.nodes[0].getblockcount(), + self.nodes[0].getbestblockhash(), + ) + dump_best_block_2 = '# mined on {}Z'.format( + datetime.datetime.fromtimestamp( + dump_time - 1, + tz=datetime.timezone.utc, + ).replace(tzinfo=None).isoformat()) + + self.log.info('Dump unencrypted wallet') result = self.nodes[0].dumpwallet(wallet_unenc_dump) assert_equal(result['filename'], wallet_unenc_dump) - found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ + found_comments, found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ read_dump(wallet_unenc_dump, addrs, [multisig_addr], None) + assert '# End of dump' in found_comments # Check that file is not corrupt + assert_equal(dump_time_str, next(c for c in found_comments if c.startswith('# * Created on'))) + assert_equal(dump_best_block_1, next(c for c in found_comments if c.startswith('# * Best block'))) + assert_equal(dump_best_block_2, next(c for c in found_comments if c.startswith('# mined on'))) assert_equal(found_legacy_addr, test_addr_count) # all keys must be in the dump assert_equal(found_p2sh_segwit_addr, test_addr_count) # all keys must be in the dump assert_equal(found_bech32_addr, test_addr_count) # all keys must be in the dump @@ -142,8 +173,12 @@ class WalletDumpTest(BitcoinTestFramework): self.nodes[0].keypoolrefill() self.nodes[0].dumpwallet(wallet_enc_dump) - found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, _ = \ + found_comments, found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, _ = \ read_dump(wallet_enc_dump, addrs, [multisig_addr], hd_master_addr_unenc) + assert '# End of dump' in found_comments # Check that file is not corrupt + assert_equal(dump_time_str, next(c for c in found_comments if c.startswith('# * Created on'))) + assert_equal(dump_best_block_1, next(c for c in found_comments if c.startswith('# * Best block'))) + assert_equal(dump_best_block_2, next(c for c in found_comments if c.startswith('# mined on'))) assert_equal(found_legacy_addr, test_addr_count) # all keys must be in the dump assert_equal(found_p2sh_segwit_addr, test_addr_count) # all keys must be in the dump assert_equal(found_bech32_addr, test_addr_count) # all keys must be in the dump diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index bc7e3cca59..6cd82ad250 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -8,7 +8,6 @@ import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - assert_equal, assert_raises_rpc_error, assert_greater_than, assert_greater_than_or_equal, @@ -27,10 +26,10 @@ class WalletEncryptionTest(BitcoinTestFramework): passphrase2 = "SecondWalletPassphrase" # Make sure the wallet isn't encrypted first - address = self.nodes[0].getnewaddress() - privkey = self.nodes[0].dumpprivkey(address) - assert_equal(privkey[:1], "c") - assert_equal(len(privkey), 52) + msg = "test message" + address = self.nodes[0].getnewaddress(address_type='legacy') + sig = self.nodes[0].signmessage(address, msg) + assert self.nodes[0].verifymessage(address, sig, msg) assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrase was called", self.nodes[0].walletpassphrase, 'ff', 1) assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrasechange was called.", self.nodes[0].walletpassphrasechange, 'ff', 'ff') @@ -39,33 +38,36 @@ class WalletEncryptionTest(BitcoinTestFramework): self.nodes[0].encryptwallet(passphrase) # Test that the wallet is encrypted - assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].signmessage, address, msg) assert_raises_rpc_error(-15, "Error: running with an encrypted wallet, but encryptwallet was called.", self.nodes[0].encryptwallet, 'ff') assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrase, '', 1) assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrasechange, '', 'ff') # Check that walletpassphrase works self.nodes[0].walletpassphrase(passphrase, 2) - assert_equal(privkey, self.nodes[0].dumpprivkey(address)) + sig = self.nodes[0].signmessage(address, msg) + assert self.nodes[0].verifymessage(address, sig, msg) # Check that the timeout is right time.sleep(3) - assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].signmessage, address, msg) # Test wrong passphrase assert_raises_rpc_error(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase + "wrong", 10) # Test walletlock self.nodes[0].walletpassphrase(passphrase, 84600) - assert_equal(privkey, self.nodes[0].dumpprivkey(address)) + sig = self.nodes[0].signmessage(address, msg) + assert self.nodes[0].verifymessage(address, sig, msg) self.nodes[0].walletlock() - assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].signmessage, address, msg) # Test passphrase changes self.nodes[0].walletpassphrasechange(passphrase, passphrase2) assert_raises_rpc_error(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase, 10) self.nodes[0].walletpassphrase(passphrase2, 10) - assert_equal(privkey, self.nodes[0].dumpprivkey(address)) + sig = self.nodes[0].signmessage(address, msg) + assert self.nodes[0].verifymessage(address, sig, msg) self.nodes[0].walletlock() # Test timeout bounds diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index 261a43472b..9dd55b4ab1 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet group functionality.""" diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 7497475b67..09f89eb59d 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Hierarchical Deterministic wallet function.""" @@ -27,17 +27,21 @@ class WalletHDTest(BitcoinTestFramework): def run_test(self): # Make sure we use hd, keep masterkeyid - masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - assert_equal(len(masterkeyid), 40) + hd_fingerprint = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['hdmasterfingerprint'] + assert_equal(len(hd_fingerprint), 8) # create an internal key change_addr = self.nodes[1].getrawchangeaddress() change_addrV= self.nodes[1].getaddressinfo(change_addr) - assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key + if self.options.descriptors: + assert_equal(change_addrV["hdkeypath"], "m/84'/1'/0'/1/0") + else: + assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key # Import a non-HD private key in the HD wallet - non_hd_add = self.nodes[0].getnewaddress() - self.nodes[1].importprivkey(self.nodes[0].dumpprivkey(non_hd_add)) + non_hd_add = 'bcrt1qmevj8zfx0wdvp05cqwkmr6mxkfx60yezwjksmt' + non_hd_key = 'cS9umN9w6cDMuRVYdbkfE4c7YUFLJRoXMfhQ569uY4odiQbVN8Rt' + self.nodes[1].importprivkey(non_hd_key) # This should be enough to keep the master key and the non-HD key self.nodes[1].backupwallet(os.path.join(self.nodes[1].datadir, "hd.bak")) @@ -48,11 +52,14 @@ class WalletHDTest(BitcoinTestFramework): self.nodes[0].generate(101) hd_add = None NUM_HD_ADDS = 10 - for i in range(NUM_HD_ADDS): + for i in range(1, NUM_HD_ADDS + 1): hd_add = self.nodes[1].getnewaddress() hd_info = self.nodes[1].getaddressinfo(hd_add) - assert_equal(hd_info["hdkeypath"], "m/0'/0'/"+str(i)+"'") - assert_equal(hd_info["hdseedid"], masterkeyid) + if self.options.descriptors: + assert_equal(hd_info["hdkeypath"], "m/84'/1'/0'/0/" + str(i)) + else: + assert_equal(hd_info["hdkeypath"], "m/0'/0'/"+str(i)+"'") + assert_equal(hd_info["hdmasterfingerprint"], hd_fingerprint) self.nodes[0].sendtoaddress(hd_add, 1) self.nodes[0].generate(1) self.nodes[0].sendtoaddress(non_hd_add, 1) @@ -61,7 +68,10 @@ class WalletHDTest(BitcoinTestFramework): # create an internal key (again) change_addr = self.nodes[1].getrawchangeaddress() change_addrV= self.nodes[1].getaddressinfo(change_addr) - assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key + if self.options.descriptors: + assert_equal(change_addrV["hdkeypath"], "m/84'/1'/0'/1/1") + else: + assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key self.sync_all() assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1) @@ -72,16 +82,19 @@ class WalletHDTest(BitcoinTestFramework): # otherwise node1 would auto-recover all funds in flag the keypool keys as used shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) - shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat")) + shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, 'wallets', "wallet.dat")) self.start_node(1) # Assert that derivation is deterministic hd_add_2 = None - for i in range(NUM_HD_ADDS): + for i in range(1, NUM_HD_ADDS + 1): hd_add_2 = self.nodes[1].getnewaddress() hd_info_2 = self.nodes[1].getaddressinfo(hd_add_2) - assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(i)+"'") - assert_equal(hd_info_2["hdseedid"], masterkeyid) + if self.options.descriptors: + assert_equal(hd_info_2["hdkeypath"], "m/84'/1'/0'/0/" + str(i)) + else: + assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(i)+"'") + assert_equal(hd_info_2["hdmasterfingerprint"], hd_fingerprint) assert_equal(hd_add, hd_add_2) connect_nodes(self.nodes[0], 1) self.sync_all() @@ -117,41 +130,45 @@ class WalletHDTest(BitcoinTestFramework): if out['value'] != 1: keypath = self.nodes[1].getaddressinfo(out['scriptPubKey']['addresses'][0])['hdkeypath'] - assert_equal(keypath[0:7], "m/0'/1'") - - # Generate a new HD seed on node 1 and make sure it is set - orig_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - self.nodes[1].sethdseed() - new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - assert orig_masterkeyid != new_masterkeyid - addr = self.nodes[1].getnewaddress() - assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is the first from the keypool - self.nodes[1].keypoolrefill(1) # Fill keypool with 1 key - - # Set a new HD seed on node 1 without flushing the keypool - new_seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress()) - orig_masterkeyid = new_masterkeyid - self.nodes[1].sethdseed(False, new_seed) - new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - assert orig_masterkeyid != new_masterkeyid - addr = self.nodes[1].getnewaddress() - assert_equal(orig_masterkeyid, self.nodes[1].getaddressinfo(addr)['hdseedid']) - assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/1\'') # Make sure the new address continues previous keypool - - # Check that the next address is from the new seed - self.nodes[1].keypoolrefill(1) - next_addr = self.nodes[1].getnewaddress() - assert_equal(new_masterkeyid, self.nodes[1].getaddressinfo(next_addr)['hdseedid']) - assert_equal(self.nodes[1].getaddressinfo(next_addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is not from previous keypool - assert next_addr != addr - - # Sethdseed parameter validity - assert_raises_rpc_error(-1, 'sethdseed', self.nodes[0].sethdseed, False, new_seed, 0) - assert_raises_rpc_error(-5, "Invalid private key", self.nodes[1].sethdseed, False, "not_wif") - assert_raises_rpc_error(-1, "JSON value is not a boolean as expected", self.nodes[1].sethdseed, "Not_bool") - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[1].sethdseed, False, True) - assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, new_seed) - assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress())) + if self.options.descriptors: + assert_equal(keypath[0:14], "m/84'/1'/0'/1/") + else: + assert_equal(keypath[0:7], "m/0'/1'") + + if not self.options.descriptors: + # Generate a new HD seed on node 1 and make sure it is set + orig_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] + self.nodes[1].sethdseed() + new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] + assert orig_masterkeyid != new_masterkeyid + addr = self.nodes[1].getnewaddress() + assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is the first from the keypool + self.nodes[1].keypoolrefill(1) # Fill keypool with 1 key + + # Set a new HD seed on node 1 without flushing the keypool + new_seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress()) + orig_masterkeyid = new_masterkeyid + self.nodes[1].sethdseed(False, new_seed) + new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] + assert orig_masterkeyid != new_masterkeyid + addr = self.nodes[1].getnewaddress() + assert_equal(orig_masterkeyid, self.nodes[1].getaddressinfo(addr)['hdseedid']) + assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/1\'') # Make sure the new address continues previous keypool + + # Check that the next address is from the new seed + self.nodes[1].keypoolrefill(1) + next_addr = self.nodes[1].getnewaddress() + assert_equal(new_masterkeyid, self.nodes[1].getaddressinfo(next_addr)['hdseedid']) + assert_equal(self.nodes[1].getaddressinfo(next_addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is not from previous keypool + assert next_addr != addr + + # Sethdseed parameter validity + assert_raises_rpc_error(-1, 'sethdseed', self.nodes[0].sethdseed, False, new_seed, 0) + assert_raises_rpc_error(-5, "Invalid private key", self.nodes[1].sethdseed, False, "not_wif") + assert_raises_rpc_error(-1, "JSON value is not a boolean as expected", self.nodes[1].sethdseed, "Not_bool") + assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[1].sethdseed, False, True) + assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, new_seed) + assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress())) if __name__ == '__main__': WalletHDTest().main () diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index b8b85b7a19..4ff7f1d525 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -40,7 +40,6 @@ Rescan = enum.Enum("Rescan", "no yes late_timestamp") class Variant(collections.namedtuple("Variant", "call data address_type rescan prune")): """Helper for importing one key and verifying scanned transactions.""" - def do_import(self, timestamp): """Call one key import RPC.""" rescan = self.rescan == Rescan.yes @@ -146,6 +145,7 @@ class ImportRescanTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + len(IMPORT_NODES) self.supports_cli = False + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -226,5 +226,6 @@ class ImportRescanTest(BitcoinTestFramework): variant.expected_txs += 1 variant.check(variant.sent_txid, variant.sent_amount, variant.confirmation_height) + if __name__ == "__main__": ImportRescanTest().main() diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py new file mode 100755 index 0000000000..55995152aa --- /dev/null +++ b/test/functional/wallet_importdescriptors.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the importdescriptors RPC. + +Test importdescriptors by generating keys on node0, importing the corresponding +descriptors on node1 and then testing the address info for the different address +variants. + +- `get_generate_key()` is called to generate keys and return the privkeys, + pubkeys and all variants of scriptPubKey and address. +- `test_importdesc()` is called to send an importdescriptors call to node1, test + success, and (if unsuccessful) test the error code and error message returned. +- `test_address()` is called to call getaddressinfo for an address on node1 + and test the values returned.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import descsum_create +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + find_vout_for_address, +) +from test_framework.wallet_util import ( + get_generate_key, + test_address, +) + +class ImportDescriptorsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.extra_args = [["-addresstype=legacy"], + ["-addresstype=bech32", "-keypool=5"] + ] + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_importdesc(self, req, success, error_code=None, error_message=None, warnings=None, wallet=None): + """Run importdescriptors and assert success""" + if warnings is None: + warnings = [] + wrpc = self.nodes[1].get_wallet_rpc('w1') + if wallet is not None: + wrpc = wallet + + result = wrpc.importdescriptors([req]) + observed_warnings = [] + if 'warnings' in result[0]: + observed_warnings = result[0]['warnings'] + assert_equal("\n".join(sorted(warnings)), "\n".join(sorted(observed_warnings))) + assert_equal(result[0]['success'], success) + if error_code is not None: + assert_equal(result[0]['error']['code'], error_code) + assert_equal(result[0]['error']['message'], error_message) + + def run_test(self): + self.log.info('Setting up wallets') + self.nodes[0].createwallet(wallet_name='w0', disable_private_keys=False) + w0 = self.nodes[0].get_wallet_rpc('w0') + + self.nodes[1].createwallet(wallet_name='w1', disable_private_keys=True, blank=True, descriptors=True) + w1 = self.nodes[1].get_wallet_rpc('w1') + assert_equal(w1.getwalletinfo()['keypoolsize'], 0) + + self.nodes[1].createwallet(wallet_name="wpriv", disable_private_keys=False, blank=True, descriptors=True) + wpriv = self.nodes[1].get_wallet_rpc("wpriv") + assert_equal(wpriv.getwalletinfo()['keypoolsize'], 0) + + self.log.info('Mining coins') + w0.generatetoaddress(101, w0.getnewaddress()) + + # RPC importdescriptors ----------------------------------------------- + + # # Test import fails if no descriptor present + key = get_generate_key() + self.log.info("Import should fail if a descriptor is not provided") + self.test_importdesc({"timestamp": "now"}, + success=False, + error_code=-8, + error_message='Descriptor not found.') + + # # Test importing of a P2PKH descriptor + key = get_generate_key() + self.log.info("Should import a p2pkh descriptor") + self.test_importdesc({"desc": descsum_create("pkh(" + key.pubkey + ")"), + "timestamp": "now", + "label": "Descriptor import test"}, + success=True) + test_address(w1, + key.p2pkh_addr, + solvable=True, + ismine=True, + labels=["Descriptor import test"]) + assert_equal(w1.getwalletinfo()['keypoolsize'], 0) + + self.log.info("Internal addresses cannot have labels") + self.test_importdesc({"desc": descsum_create("pkh(" + key.pubkey + ")"), + "timestamp": "now", + "internal": True, + "label": "Descriptor import test"}, + success=False, + error_code=-8, + error_message="Internal addresses should not have a label") + + # # Test importing of a P2SH-P2WPKH descriptor + key = get_generate_key() + self.log.info("Should not import a p2sh-p2wpkh descriptor without checksum") + self.test_importdesc({"desc": "sh(wpkh(" + key.pubkey + "))", + "timestamp": "now" + }, + success=False, + error_code=-5, + error_message="Missing checksum") + + self.log.info("Should not import a p2sh-p2wpkh descriptor that has range specified") + self.test_importdesc({"desc": descsum_create("sh(wpkh(" + key.pubkey + "))"), + "timestamp": "now", + "range": 1, + }, + success=False, + error_code=-8, + error_message="Range should not be specified for an un-ranged descriptor") + + self.log.info("Should not import a p2sh-p2wpkh descriptor and have it set to active") + self.test_importdesc({"desc": descsum_create("sh(wpkh(" + key.pubkey + "))"), + "timestamp": "now", + "active": True, + }, + success=False, + error_code=-8, + error_message="Active descriptors must be ranged") + + self.log.info("Should import a (non-active) p2sh-p2wpkh descriptor") + self.test_importdesc({"desc": descsum_create("sh(wpkh(" + key.pubkey + "))"), + "timestamp": "now", + "active": False, + }, + success=True) + assert_equal(w1.getwalletinfo()['keypoolsize'], 0) + + test_address(w1, + key.p2sh_p2wpkh_addr, + ismine=True, + solvable=True) + + # # Test importing of a multisig descriptor + key1 = get_generate_key() + key2 = get_generate_key() + self.log.info("Should import a 1-of-2 bare multisig from descriptor") + self.test_importdesc({"desc": descsum_create("multi(1," + key1.pubkey + "," + key2.pubkey + ")"), + "timestamp": "now"}, + success=True) + self.log.info("Should not treat individual keys from the imported bare multisig as watchonly") + test_address(w1, + key1.p2pkh_addr, + ismine=False) + + # # Test ranged descriptors + xpriv = "tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg" + xpub = "tpubD6NzVbkrYhZ4YNXVQbNhMK1WqguFsUXceaVJKbmno2aZ3B6QfbMeraaYvnBSGpV3vxLyTTK9DYT1yoEck4XUScMzXoQ2U2oSmE2JyMedq3H" + addresses = ["2N7yv4p8G8yEaPddJxY41kPihnWvs39qCMf", "2MsHxyb2JS3pAySeNUsJ7mNnurtpeenDzLA"] # hdkeypath=m/0'/0'/0' and 1' + addresses += ["bcrt1qrd3n235cj2czsfmsuvqqpr3lu6lg0ju7scl8gn", "bcrt1qfqeppuvj0ww98r6qghmdkj70tv8qpchehegrg8"] # wpkh subscripts corresponding to the above addresses + desc = "sh(wpkh(" + xpub + "/0/0/*" + "))" + + self.log.info("Ranged descriptors cannot have labels") + self.test_importdesc({"desc":descsum_create(desc), + "timestamp": "now", + "range": [0, 100], + "label": "test"}, + success=False, + error_code=-8, + error_message='Ranged descriptors should not have a label') + + self.log.info("Private keys required for private keys enabled wallet") + self.test_importdesc({"desc":descsum_create(desc), + "timestamp": "now", + "range": [0, 100]}, + success=False, + error_code=-4, + error_message='Cannot import descriptor without private keys to a wallet with private keys enabled', + wallet=wpriv) + + self.log.info("Ranged descriptor import should warn without a specified range") + self.test_importdesc({"desc": descsum_create(desc), + "timestamp": "now"}, + success=True, + warnings=['Range not given, using default keypool range']) + assert_equal(w1.getwalletinfo()['keypoolsize'], 0) + + # # Test importing of a ranged descriptor with xpriv + self.log.info("Should not import a ranged descriptor that includes xpriv into a watch-only wallet") + desc = "sh(wpkh(" + xpriv + "/0'/0'/*'" + "))" + self.test_importdesc({"desc": descsum_create(desc), + "timestamp": "now", + "range": 1}, + success=False, + error_code=-4, + error_message='Cannot import private keys to a wallet with private keys disabled') + for address in addresses: + test_address(w1, + address, + ismine=False, + solvable=False) + + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": -1}, + success=False, error_code=-8, error_message='End of range is too high') + + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": [-1, 10]}, + success=False, error_code=-8, error_message='Range should be greater or equal than 0') + + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": [(2 << 31 + 1) - 1000000, (2 << 31 + 1)]}, + success=False, error_code=-8, error_message='End of range is too high') + + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": [2, 1]}, + success=False, error_code=-8, error_message='Range specified as [begin,end] must not have begin after end') + + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": [0, 1000001]}, + success=False, error_code=-8, error_message='Range is too large') + + # Make sure ranged imports import keys in order + w1 = self.nodes[1].get_wallet_rpc('w1') + self.log.info('Key ranges should be imported in order') + xpub = "tpubDAXcJ7s7ZwicqjprRaEWdPoHKrCS215qxGYxpusRLLmJuT69ZSicuGdSfyvyKpvUNYBW1s2U3NSrT6vrCYB9e6nZUEvrqnwXPF8ArTCRXMY" + addresses = [ + 'bcrt1qtmp74ayg7p24uslctssvjm06q5phz4yrxucgnv', # m/0'/0'/0 + 'bcrt1q8vprchan07gzagd5e6v9wd7azyucksq2xc76k8', # m/0'/0'/1 + 'bcrt1qtuqdtha7zmqgcrr26n2rqxztv5y8rafjp9lulu', # m/0'/0'/2 + 'bcrt1qau64272ymawq26t90md6an0ps99qkrse58m640', # m/0'/0'/3 + 'bcrt1qsg97266hrh6cpmutqen8s4s962aryy77jp0fg0', # m/0'/0'/4 + ] + + self.test_importdesc({'desc': descsum_create('wpkh([80002067/0h/0h]' + xpub + '/*)'), + 'active': True, + 'range' : [0, 2], + 'timestamp': 'now' + }, + success=True) + self.test_importdesc({'desc': descsum_create('sh(wpkh([abcdef12/0h/0h]' + xpub + '/*))'), + 'active': True, + 'range' : [0, 2], + 'timestamp': 'now' + }, + success=True) + self.test_importdesc({'desc': descsum_create('pkh([12345678/0h/0h]' + xpub + '/*)'), + 'active': True, + 'range' : [0, 2], + 'timestamp': 'now' + }, + success=True) + + assert_equal(w1.getwalletinfo()['keypoolsize'], 5 * 3) + for i, expected_addr in enumerate(addresses): + received_addr = w1.getnewaddress('', 'bech32') + assert_raises_rpc_error(-4, 'This wallet has no available keys', w1.getrawchangeaddress, 'bech32') + assert_equal(received_addr, expected_addr) + bech32_addr_info = w1.getaddressinfo(received_addr) + assert_equal(bech32_addr_info['desc'][:23], 'wpkh([80002067/0\'/0\'/{}]'.format(i)) + + shwpkh_addr = w1.getnewaddress('', 'p2sh-segwit') + shwpkh_addr_info = w1.getaddressinfo(shwpkh_addr) + assert_equal(shwpkh_addr_info['desc'][:26], 'sh(wpkh([abcdef12/0\'/0\'/{}]'.format(i)) + + pkh_addr = w1.getnewaddress('', 'legacy') + pkh_addr_info = w1.getaddressinfo(pkh_addr) + assert_equal(pkh_addr_info['desc'][:22], 'pkh([12345678/0\'/0\'/{}]'.format(i)) + + assert_equal(w1.getwalletinfo()['keypoolsize'], 4 * 3) # After retrieving a key, we don't refill the keypool again, so it's one less for each address type + w1.keypoolrefill() + assert_equal(w1.getwalletinfo()['keypoolsize'], 5 * 3) + + # Check active=False default + self.log.info('Check imported descriptors are not active by default') + self.test_importdesc({'desc': descsum_create('pkh([12345678/0h/0h]' + xpub + '/*)'), + 'range' : [0, 2], + 'timestamp': 'now', + 'internal': True + }, + success=True) + assert_raises_rpc_error(-4, 'This wallet has no available keys', w1.getrawchangeaddress, 'legacy') + + # # Test importing a descriptor containing a WIF private key + wif_priv = "cTe1f5rdT8A8DFgVWTjyPwACsDPJM9ff4QngFxUixCSvvbg1x6sh" + address = "2MuhcG52uHPknxDgmGPsV18jSHFBnnRgjPg" + desc = "sh(wpkh(" + wif_priv + "))" + self.log.info("Should import a descriptor with a WIF private key as spendable") + self.test_importdesc({"desc": descsum_create(desc), + "timestamp": "now"}, + success=True, + wallet=wpriv) + test_address(wpriv, + address, + solvable=True, + ismine=True) + txid = w0.sendtoaddress(address, 49.99995540) + w0.generatetoaddress(6, w0.getnewaddress()) + self.sync_blocks() + tx = wpriv.createrawtransaction([{"txid": txid, "vout": 0}], {w0.getnewaddress(): 49.999}) + signed_tx = wpriv.signrawtransactionwithwallet(tx) + w1.sendrawtransaction(signed_tx['hex']) + + # Make sure that we can use import and use multisig as addresses + self.log.info('Test that multisigs can be imported, signed for, and getnewaddress\'d') + self.nodes[1].createwallet(wallet_name="wmulti_priv", disable_private_keys=False, blank=True, descriptors=True) + wmulti_priv = self.nodes[1].get_wallet_rpc("wmulti_priv") + assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 0) + + self.test_importdesc({"desc":"wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/0h/0h/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/0h/0h/*,tprv8ZgxMBicQKsPeonDt8Ka2mrQmHa61hQ5FQCsvWBTpSNzBFgM58cV2EuXNAHF14VawVpznnme3SuTbA62sGriwWyKifJmXntfNeK7zeqMCj1/84h/0h/0h/*))#m2sr93jn", + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now"}, + success=True, + wallet=wmulti_priv) + self.test_importdesc({"desc":"wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/1h/0h/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/1h/0h/*,tprv8ZgxMBicQKsPeonDt8Ka2mrQmHa61hQ5FQCsvWBTpSNzBFgM58cV2EuXNAHF14VawVpznnme3SuTbA62sGriwWyKifJmXntfNeK7zeqMCj1/84h/1h/0h/*))#q3sztvx5", + "active": True, + "internal" : True, + "range": 1000, + "next_index": 0, + "timestamp": "now"}, + success=True, + wallet=wmulti_priv) + + assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 1001) # Range end (1000) is inclusive, so 1001 addresses generated + addr = wmulti_priv.getnewaddress('', 'bech32') + assert_equal(addr, 'bcrt1qdt0qy5p7dzhxzmegnn4ulzhard33s2809arjqgjndx87rv5vd0fq2czhy8') # Derived at m/84'/0'/0'/0 + change_addr = wmulti_priv.getrawchangeaddress('bech32') + assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') + assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 1000) + txid = w0.sendtoaddress(addr, 10) + self.nodes[0].generate(6) + send_txid = wmulti_priv.sendtoaddress(w0.getnewaddress(), 8) + decoded = wmulti_priv.decoderawtransaction(wmulti_priv.gettransaction(send_txid)['hex']) + assert_equal(len(decoded['vin'][0]['txinwitness']), 4) + self.nodes[0].generate(6) + self.sync_all() + + self.nodes[1].createwallet(wallet_name="wmulti_pub", disable_private_keys=True, blank=True, descriptors=True) + wmulti_pub = self.nodes[1].get_wallet_rpc("wmulti_pub") + assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 0) + + self.test_importdesc({"desc":"wsh(multi(2,[7b2d0242/84h/0h/0h]tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*,[59b09cd6/84h/0h/0h]tpubDDBF2BTR6s8drwrfDei8WxtckGuSm1cyoKxYY1QaKSBFbHBYQArWhHPA6eJrzZej6nfHGLSURYSLHr7GuYch8aY5n61tGqgn8b4cXrMuoPH/*,[e81a0532/84h/0h/0h]tpubDCsWoW1kuQB9kG5MXewHqkbjPtqPueRnXju7uM2NK7y3JYb2ajAZ9EiuZXNNuE4661RAfriBWhL8UsnAPpk8zrKKnZw1Ug7X4oHgMdZiU4E/*))#tsry0s5e", + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now"}, + success=True, + wallet=wmulti_pub) + self.test_importdesc({"desc":"wsh(multi(2,[7b2d0242/84h/1h/0h]tpubDCXqdwWZcszwqYJSnZp8eARkxGJfHAk23KDxbztV4BbschfaTfYLTcSkSJ3TN64dRqwa1rnFUScsYormKkGqNbbPwkorQimVevXjxzUV9Gf/*,[59b09cd6/84h/1h/0h]tpubDCYfZY2ceyHzYzMMVPt9MNeiqtQ2T7Uyp9QSFwYXh8Vi9iJFYXcuphJaGXfF3jUQJi5Y3GMNXvM11gaL4txzZgNGK22BFAwMXynnzv4z2Jh/*,[e81a0532/84h/1h/0h]tpubDC6UGqnsQStngYuGD4MKsMy7eD1Yg9NTJfPdvjdG2JE5oZ7EsSL3WHg4Gsw2pR5K39ZwJ46M1wZayhedVdQtMGaUhq5S23PH6fnENK3V1sb/*))#c08a2rzv", + "active": True, + "internal" : True, + "range": 1000, + "next_index": 0, + "timestamp": "now"}, + success=True, + wallet=wmulti_pub) + + assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 1000) # The first one was already consumed by previous import and is detected as used + addr = wmulti_pub.getnewaddress('', 'bech32') + assert_equal(addr, 'bcrt1qp8s25ckjl7gr6x2q3dx3tn2pytwp05upkjztk6ey857tt50r5aeqn6mvr9') # Derived at m/84'/0'/0'/1 + change_addr = wmulti_pub.getrawchangeaddress('bech32') + assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') + assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 999) + txid = w0.sendtoaddress(addr, 10) + vout = find_vout_for_address(self.nodes[0], txid, addr) + self.nodes[0].generate(6) + self.sync_all() + assert_equal(wmulti_pub.getbalance(), wmulti_priv.getbalance()) + + self.log.info("Multisig with distributed keys") + self.nodes[1].createwallet(wallet_name="wmulti_priv1", descriptors=True) + wmulti_priv1 = self.nodes[1].get_wallet_rpc("wmulti_priv1") + res = wmulti_priv1.importdescriptors([ + { + "desc": descsum_create("wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/0h/0h/*,[59b09cd6/84h/0h/0h]tpubDDBF2BTR6s8drwrfDei8WxtckGuSm1cyoKxYY1QaKSBFbHBYQArWhHPA6eJrzZej6nfHGLSURYSLHr7GuYch8aY5n61tGqgn8b4cXrMuoPH/*,[e81a0532/84h/0h/0h]tpubDCsWoW1kuQB9kG5MXewHqkbjPtqPueRnXju7uM2NK7y3JYb2ajAZ9EiuZXNNuE4661RAfriBWhL8UsnAPpk8zrKKnZw1Ug7X4oHgMdZiU4E/*))"), + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }, + { + "desc": descsum_create("wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/1h/0h/*,[59b09cd6/84h/1h/0h]tpubDCYfZY2ceyHzYzMMVPt9MNeiqtQ2T7Uyp9QSFwYXh8Vi9iJFYXcuphJaGXfF3jUQJi5Y3GMNXvM11gaL4txzZgNGK22BFAwMXynnzv4z2Jh/*,[e81a0532/84h/1h/0h]tpubDC6UGqnsQStngYuGD4MKsMy7eD1Yg9NTJfPdvjdG2JE5oZ7EsSL3WHg4Gsw2pR5K39ZwJ46M1wZayhedVdQtMGaUhq5S23PH6fnENK3V1sb/*))"), + "active": True, + "internal" : True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }]) + assert_equal(res[0]['success'], True) + assert_equal(res[0]['warnings'][0], 'Not all private keys provided. Some wallet functionality may return unexpected errors') + assert_equal(res[1]['success'], True) + assert_equal(res[1]['warnings'][0], 'Not all private keys provided. Some wallet functionality may return unexpected errors') + + self.nodes[1].createwallet(wallet_name='wmulti_priv2', blank=True, descriptors=True) + wmulti_priv2 = self.nodes[1].get_wallet_rpc('wmulti_priv2') + res = wmulti_priv2.importdescriptors([ + { + "desc": descsum_create("wsh(multi(2,[7b2d0242/84h/0h/0h]tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/0h/0h/*,[e81a0532/84h/0h/0h]tpubDCsWoW1kuQB9kG5MXewHqkbjPtqPueRnXju7uM2NK7y3JYb2ajAZ9EiuZXNNuE4661RAfriBWhL8UsnAPpk8zrKKnZw1Ug7X4oHgMdZiU4E/*))"), + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }, + { + "desc": descsum_create("wsh(multi(2,[7b2d0242/84h/1h/0h]tpubDCXqdwWZcszwqYJSnZp8eARkxGJfHAk23KDxbztV4BbschfaTfYLTcSkSJ3TN64dRqwa1rnFUScsYormKkGqNbbPwkorQimVevXjxzUV9Gf/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/1h/0h/*,[e81a0532/84h/1h/0h]tpubDC6UGqnsQStngYuGD4MKsMy7eD1Yg9NTJfPdvjdG2JE5oZ7EsSL3WHg4Gsw2pR5K39ZwJ46M1wZayhedVdQtMGaUhq5S23PH6fnENK3V1sb/*))"), + "active": True, + "internal" : True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }]) + assert_equal(res[0]['success'], True) + assert_equal(res[0]['warnings'][0], 'Not all private keys provided. Some wallet functionality may return unexpected errors') + assert_equal(res[1]['success'], True) + assert_equal(res[1]['warnings'][0], 'Not all private keys provided. Some wallet functionality may return unexpected errors') + + rawtx = self.nodes[1].createrawtransaction([{'txid': txid, 'vout': vout}], {w0.getnewaddress(): 9.999}) + tx_signed_1 = wmulti_priv1.signrawtransactionwithwallet(rawtx) + assert_equal(tx_signed_1['complete'], False) + tx_signed_2 = wmulti_priv2.signrawtransactionwithwallet(tx_signed_1['hex']) + assert_equal(tx_signed_2['complete'], True) + self.nodes[1].sendrawtransaction(tx_signed_2['hex']) + + self.log.info("Combo descriptors cannot be active") + self.test_importdesc({"desc": descsum_create("combo(tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*)"), + "active": True, + "range": 1, + "timestamp": "now"}, + success=False, + error_code=-4, + error_message="Combo descriptors cannot be set to active") + + self.log.info("Descriptors with no type cannot be active") + self.test_importdesc({"desc": descsum_create("pk(tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*)"), + "active": True, + "range": 1, + "timestamp": "now"}, + success=True, + warnings=["Unknown output type, cannot set descriptor to active."]) + +if __name__ == '__main__': + ImportDescriptorsTest().main() diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index f152fcd1a4..bd4fcdabcf 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -32,6 +32,7 @@ from test_framework.wallet_util import ( test_address, ) + class ImportMultiTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -51,7 +52,7 @@ class ImportMultiTest(BitcoinTestFramework): result = self.nodes[1].importmulti([req]) observed_warnings = [] if 'warnings' in result[0]: - observed_warnings = result[0]['warnings'] + observed_warnings = result[0]['warnings'] assert_equal("\n".join(sorted(warnings)), "\n".join(sorted(observed_warnings))) assert_equal(result[0]['success'], success) if error_code is not None: @@ -63,6 +64,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[0].generate(1) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] + self.nodes[1].syncwithvalidationinterfacequeue() node0_address1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) @@ -257,6 +259,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] + self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -277,6 +280,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] + self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh with respective redeem script") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -297,6 +301,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] + self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh with respective redeem script and private keys") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -322,6 +327,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] + self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh with respective redeem script and private keys") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -851,5 +857,6 @@ class ImportMultiTest(BitcoinTestFramework): addr = wrpc.getnewaddress('', 'bech32') assert_equal(addr, addresses[i]) + if __name__ == '__main__': ImportMultiTest().main() diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index e3aeb61197..40a2b3ab6a 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -5,6 +5,7 @@ """Test the wallet keypool and interaction with wallet encryption/locking.""" import time +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error @@ -21,16 +22,63 @@ class KeyPoolTest(BitcoinTestFramework): addr_before_encrypting = nodes[0].getnewaddress() addr_before_encrypting_data = nodes[0].getaddressinfo(addr_before_encrypting) wallet_info_old = nodes[0].getwalletinfo() - assert addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid'] + if not self.options.descriptors: + assert addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid'] # Encrypt wallet and wait to terminate nodes[0].encryptwallet('test') + if self.options.descriptors: + # Import hardened derivation only descriptors + nodes[0].walletpassphrase('test', 10) + nodes[0].importdescriptors([ + { + "desc": "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/*h)#y4dfsj7n", + "timestamp": "now", + "range": [0,0], + "active": True + }, + { + "desc": "pkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1h/*h)#a0nyvl0k", + "timestamp": "now", + "range": [0,0], + "active": True + }, + { + "desc": "sh(wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/2h/*h))#lmeu2axg", + "timestamp": "now", + "range": [0,0], + "active": True + }, + { + "desc": "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/3h/*h)#jkl636gm", + "timestamp": "now", + "range": [0,0], + "active": True, + "internal": True + }, + { + "desc": "pkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/4h/*h)#l3crwaus", + "timestamp": "now", + "range": [0,0], + "active": True, + "internal": True + }, + { + "desc": "sh(wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/5h/*h))#qg8wa75f", + "timestamp": "now", + "range": [0,0], + "active": True, + "internal": True + } + ]) + nodes[0].walletlock() # Keep creating keys addr = nodes[0].getnewaddress() addr_data = nodes[0].getaddressinfo(addr) wallet_info = nodes[0].getwalletinfo() - assert addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid'] - assert addr_data['hdseedid'] == wallet_info['hdseedid'] + assert addr_before_encrypting_data['hdmasterfingerprint'] != addr_data['hdmasterfingerprint'] + if not self.options.descriptors: + assert addr_data['hdseedid'] == wallet_info['hdseedid'] assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) # put six (plus 2) new keys in the keypool (100% external-, +100% internal-keys, 1 in min) @@ -38,8 +86,12 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0].keypoolrefill(6) nodes[0].walletlock() wi = nodes[0].getwalletinfo() - assert_equal(wi['keypoolsize_hd_internal'], 6) - assert_equal(wi['keypoolsize'], 6) + if self.options.descriptors: + assert_equal(wi['keypoolsize_hd_internal'], 18) + assert_equal(wi['keypoolsize'], 18) + else: + assert_equal(wi['keypoolsize_hd_internal'], 6) + assert_equal(wi['keypoolsize'], 6) # drain the internal keys nodes[0].getrawchangeaddress() @@ -53,12 +105,12 @@ class KeyPoolTest(BitcoinTestFramework): assert_raises_rpc_error(-12, "Keypool ran out", nodes[0].getrawchangeaddress) # drain the external keys - addr.add(nodes[0].getnewaddress()) - addr.add(nodes[0].getnewaddress()) - addr.add(nodes[0].getnewaddress()) - addr.add(nodes[0].getnewaddress()) - addr.add(nodes[0].getnewaddress()) - addr.add(nodes[0].getnewaddress()) + addr.add(nodes[0].getnewaddress(address_type="bech32")) + addr.add(nodes[0].getnewaddress(address_type="bech32")) + addr.add(nodes[0].getnewaddress(address_type="bech32")) + addr.add(nodes[0].getnewaddress(address_type="bech32")) + addr.add(nodes[0].getnewaddress(address_type="bech32")) + addr.add(nodes[0].getnewaddress(address_type="bech32")) assert len(addr) == 6 # the next one should fail assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) @@ -79,8 +131,62 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0].walletpassphrase('test', 100) nodes[0].keypoolrefill(100) wi = nodes[0].getwalletinfo() - assert_equal(wi['keypoolsize_hd_internal'], 100) - assert_equal(wi['keypoolsize'], 100) + if self.options.descriptors: + assert_equal(wi['keypoolsize_hd_internal'], 300) + assert_equal(wi['keypoolsize'], 300) + else: + assert_equal(wi['keypoolsize_hd_internal'], 100) + assert_equal(wi['keypoolsize'], 100) + + # create a blank wallet + nodes[0].createwallet(wallet_name='w2', blank=True, disable_private_keys=True) + w2 = nodes[0].get_wallet_rpc('w2') + + # refer to initial wallet as w1 + w1 = nodes[0].get_wallet_rpc('') + + # import private key and fund it + address = addr.pop() + desc = w1.getaddressinfo(address)['desc'] + if self.options.descriptors: + res = w2.importdescriptors([{'desc': desc, 'timestamp': 'now'}]) + else: + res = w2.importmulti([{'desc': desc, 'timestamp': 'now'}]) + assert_equal(res[0]['success'], True) + w1.walletpassphrase('test', 100) + + res = w1.sendtoaddress(address=address, amount=0.00010000) + nodes[0].generate(1) + destination = addr.pop() + + # Using a fee rate (10 sat / byte) well above the minimum relay rate + # creating a 5,000 sat transaction with change should not be possible + assert_raises_rpc_error(-4, "Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.", w2.walletcreatefundedpsbt, inputs=[], outputs=[{addr.pop(): 0.00005000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + + # creating a 10,000 sat transaction without change, with a manual input, should still be possible + res = w2.walletcreatefundedpsbt(inputs=w2.listunspent(), outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + assert_equal("psbt" in res, True) + + # creating a 10,000 sat transaction without change should still be possible + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + assert_equal("psbt" in res, True) + # should work without subtractFeeFromOutputs if the exact fee is subtracted from the amount + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008900}], options={"feeRate": 0.00010}) + assert_equal("psbt" in res, True) + + # dust change should be removed + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008800}], options={"feeRate": 0.00010}) + assert_equal("psbt" in res, True) + + # create a transaction without change at the maximum fee rate, such that the output is still spendable: + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.0008824}) + assert_equal("psbt" in res, True) + assert_equal(res["fee"], Decimal("0.00009706")) + + # creating a 10,000 sat transaction with a manual change address should be possible + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010, "changeAddress": addr.pop()}) + assert_equal("psbt" in res, True) + if __name__ == '__main__': KeyPoolTest().main() diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index 829633a050..102ed23fba 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -79,7 +79,15 @@ class KeypoolRestoreTest(BitcoinTestFramework): assert_equal(self.nodes[idx].getbalance(), 15) assert_equal(self.nodes[idx].listtransactions()[0]['category'], "receive") # Check that we have marked all keys up to the used keypool key as used - assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress())['hdkeypath'], "m/0'/0'/110'") + if self.options.descriptors: + if output_type == 'legacy': + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/44'/1'/0'/0/110") + elif output_type == 'p2sh-segwit': + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/49'/1'/0'/0/110") + elif output_type == 'bech32': + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/84'/1'/0'/0/110") + else: + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/0'/0'/110'") if __name__ == '__main__': diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index 337d2e55d9..f8d1720469 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -115,15 +115,16 @@ class WalletLabelsTest(BitcoinTestFramework): assert_raises_rpc_error(-11, "No addresses with label", node.getaddressesbylabel, "") # Check that addmultisigaddress can assign labels. - for label in labels: - addresses = [] - for x in range(10): - addresses.append(node.getnewaddress()) - multisig_address = node.addmultisigaddress(5, addresses, label.name)['address'] - label.add_address(multisig_address) - label.purpose[multisig_address] = "send" - label.verify(node) - node.generate(101) + if not self.options.descriptors: + for label in labels: + addresses = [] + for x in range(10): + addresses.append(node.getnewaddress()) + multisig_address = node.addmultisigaddress(5, addresses, label.name)['address'] + label.add_address(multisig_address) + label.purpose[multisig_address] = "send" + label.verify(node) + node.generate(101) # Check that setlabel can change the label of an address from a # different label. diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 8c44a070b8..8ff663ccd2 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -97,6 +97,8 @@ class ListTransactionsTest(BitcoinTestFramework): txid = self.nodes[1].sendtoaddress(multisig["address"], 0.1) self.nodes[1].generate(1) self.sync_all() + assert_equal(len(self.nodes[0].listtransactions(label="watchonly", include_watchonly=True)), 1) + assert_equal(len(self.nodes[0].listtransactions(dummy="watchonly", include_watchonly=True)), 1) assert len(self.nodes[0].listtransactions(label="watchonly", count=100, include_watchonly=False)) == 0 assert_array_result(self.nodes[0].listtransactions(label="watchonly", count=100, include_watchonly=True), {"category": "receive", "amount": Decimal("0.1")}, diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index a2c502f280..c569416292 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiwallet. Verify that a bitcoind node can load multiple wallet files """ +from decimal import Decimal import os import shutil import time @@ -125,10 +126,6 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(['-salvagewallet', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file") self.nodes[0].assert_start_raises_init_error(['-salvagewallet=1', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file") - self.log.info("Do not allow -upgradewallet with multiwallet") - self.nodes[0].assert_start_raises_init_error(['-upgradewallet', '-wallet=w1', '-wallet=w2'], "Error: -upgradewallet is only allowed with a single wallet file") - self.nodes[0].assert_start_raises_init_error(['-upgradewallet=1', '-wallet=w1', '-wallet=w2'], "Error: -upgradewallet is only allowed with a single wallet file") - # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) @@ -193,9 +190,9 @@ class MultiWalletTest(BitcoinTestFramework): self.log.info('Check for per-wallet settxfee call') assert_equal(w1.getwalletinfo()['paytxfee'], 0) assert_equal(w2.getwalletinfo()['paytxfee'], 0) - w2.settxfee(4.0) + w2.settxfee(0.001) assert_equal(w1.getwalletinfo()['paytxfee'], 0) - assert_equal(w2.getwalletinfo()['paytxfee'], 4.0) + assert_equal(w2.getwalletinfo()['paytxfee'], Decimal('0.00100000')) self.log.info("Test dynamic wallet loading") @@ -332,18 +329,5 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].unloadwallet(wallet) self.nodes[1].loadwallet(wallet) - # Fail to load if wallet is downgraded - shutil.copytree(os.path.join(self.options.data_wallets_dir, 'high_minversion'), wallet_dir('high_minversion')) - self.restart_node(0, extra_args=['-upgradewallet={}'.format(FEATURE_LATEST)]) - assert {'name': 'high_minversion'} in self.nodes[0].listwalletdir()['wallets'] - self.log.info("Fail -upgradewallet that results in downgrade") - assert_raises_rpc_error( - -4, - 'Wallet loading failed: Error loading {}: Wallet requires newer version of {}'.format( - wallet_dir('high_minversion', 'wallet.dat'), self.config['environment']['PACKAGE_NAME']), - lambda: self.nodes[0].loadwallet(filename='high_minversion'), - ) - - if __name__ == '__main__': MultiWalletTest().main() diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index d122e3db52..db5902f820 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet resends transactions periodically.""" diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index 99559090ee..ad23206c90 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -29,7 +29,7 @@ class TxnMallTest(BitcoinTestFramework): def setup_network(self): # Start with split network: - super(TxnMallTest, self).setup_network() + super().setup_network() disconnect_nodes(self.nodes[1], 2) disconnect_nodes(self.nodes[2], 1) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index bb93060739..e2454c4237 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -1,16 +1,17 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run fuzz test targets. """ +from concurrent.futures import ThreadPoolExecutor, as_completed import argparse import configparser +import logging import os -import sys import subprocess -import logging +import sys def main(): @@ -36,6 +37,12 @@ def main(): help="A comma-separated list of targets to exclude", ) parser.add_argument( + '--par', + type=int, + default=4, + help='How many targets to merge or execute in parallel.', + ) + parser.add_argument( 'seed_dir', help='The seed corpus to run on (must contain subfolders for each fuzz target).', ) @@ -124,40 +131,53 @@ def main(): logging.error("subprocess timed out: Currently only libFuzzer is supported") sys.exit(1) - if args.m_dir: - merge_inputs( + with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: + if args.m_dir: + merge_inputs( + fuzz_pool=fuzz_pool, + corpus=args.seed_dir, + test_list=test_list_selection, + build_dir=config["environment"]["BUILDDIR"], + merge_dir=args.m_dir, + ) + return + + run_once( + fuzz_pool=fuzz_pool, corpus=args.seed_dir, test_list=test_list_selection, build_dir=config["environment"]["BUILDDIR"], - merge_dir=args.m_dir, + use_valgrind=args.valgrind, ) - return - - run_once( - corpus=args.seed_dir, - test_list=test_list_selection, - build_dir=config["environment"]["BUILDDIR"], - use_valgrind=args.valgrind, - ) -def merge_inputs(*, corpus, test_list, build_dir, merge_dir): +def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): logging.info("Merge the inputs in the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) + jobs = [] for t in test_list: args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', t), '-merge=1', + '-use_value_profile=1', # Also done by oss-fuzz https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 os.path.join(corpus, t), os.path.join(merge_dir, t), ] os.makedirs(os.path.join(corpus, t), exist_ok=True) os.makedirs(os.path.join(merge_dir, t), exist_ok=True) - logging.debug('Run {} with args {}'.format(t, args)) - output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr - logging.debug('Output: {}'.format(output)) + def job(t, args): + output = 'Run {} with args {}\n'.format(t, " ".join(args)) + output += subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr + logging.debug(output) + + jobs.append(fuzz_pool.submit(job, t, args)) + + for future in as_completed(jobs): + future.result() -def run_once(*, corpus, test_list, build_dir, use_valgrind): + +def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): + jobs = [] for t in test_list: corpus_path = os.path.join(corpus, t) os.makedirs(corpus_path, exist_ok=True) @@ -168,10 +188,18 @@ def run_once(*, corpus, test_list, build_dir, use_valgrind): ] if use_valgrind: args = ['valgrind', '--quiet', '--error-exitcode=1'] + args - logging.debug('Run {} with args {}'.format(t, args)) - result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True) - output = result.stderr - logging.debug('Output: {}'.format(output)) + + def job(t, args): + output = 'Run {} with args {}'.format(t, args) + result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True) + output += result.stderr + return output, result + + jobs.append(fuzz_pool.submit(job, t, args)) + + for future in as_completed(jobs): + output, result = future.result() + logging.debug(output) try: result.check_returncode() except subprocess.CalledProcessError as e: @@ -179,7 +207,7 @@ def run_once(*, corpus, test_list, build_dir, use_valgrind): logging.info(e.stdout) if e.stderr: logging.info(e.stderr) - logging.info("Target \"{}\" failed with exit code {}: {}".format(t, e.returncode, " ".join(args))) + logging.info("Target \"{}\" failed with exit code {}".format(" ".join(result.args), e.returncode)) sys.exit(1) diff --git a/test/lint/extended-lint-all.sh b/test/lint/extended-lint-all.sh index 65c51e02f5..be5d9db4a9 100755 --- a/test/lint/extended-lint-all.sh +++ b/test/lint/extended-lint-all.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/extended-lint-cppcheck.sh b/test/lint/extended-lint-cppcheck.sh index ae18d74ebf..20021d8605 100755 --- a/test/lint/extended-lint-cppcheck.sh +++ b/test/lint/extended-lint-cppcheck.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index bbd94dd6c7..6bd02d45ac 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh index 184c3682c8..cb3ec09ae6 100755 --- a/test/lint/lint-format-strings.sh +++ b/test/lint/lint-format-strings.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index 1cece6a525..bd9c8337ac 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 70410d7405..e2bb403c4d 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh index 773855bed1..7257919c98 100755 --- a/test/lint/lint-python-utf8-encoding.sh +++ b/test/lint/lint-python-utf8-encoding.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-shebang.sh b/test/lint/lint-shebang.sh index a666fdfecf..a5c8aa42b2 100755 --- a/test/lint/lint-shebang.sh +++ b/test/lint/lint-shebang.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index d31ea14ea8..2bb76ec286 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # |