diff options
400 files changed, 4621 insertions, 1769 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 777eebd2c3..eeb2d6590b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,10 +16,8 @@ environment: VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' cache: - C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt -- C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in - C:\Qt5.9.8_x64_static_vs2019 install: -- cmd: pip install --quiet git+https://github.com/frerich/clcache.git@v4.2.0 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq # Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is: @@ -46,7 +44,6 @@ install: } c:\tools\vcpkg\vcpkg integrate install before_build: -- ps: clcache -M 536870912 # Powershell block below is to download and extract the Qt static libraries. The pseudo code is: # 1. If the Qt destination directory exists assume it is correct and do nothing. To # force a fresh install of the packages delete the job's appveyor cache. @@ -72,21 +69,20 @@ before_build: Write-Host "Qt binaries already present."; } - cmd: python build_msvc\msvc-autogen.py -- ps: Start-Process clcache-server -- ps: fsutil behavior set disablelastaccess 0 # Enable Access time feature on Windows (for clcache) build_script: -- cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\bitcoin.sln /m /v:q /nologo +- cmd: msbuild /p:TrackFileAccess=false build_msvc\bitcoin.sln /m /v:q /nologo after_build: -- ps: fsutil behavior set disablelastaccess 1 # Disable Access time feature on Windows (better performance) -- ps: clcache -z #- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe test_script: -- cmd: src\test_bitcoin.exe -k stdout -e stdout 2> NUL +- cmd: src\test_bitcoin.exe -l test_suite - cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL - ps: python test\util\bitcoin-util-test.py - cmd: python test\util\rpcauth-test.py # Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted. -- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation +# functional tests disabled for now. See +# https://github.com/bitcoin/bitcoin/pull/18626#issuecomment-613396202 +# https://github.com/bitcoin/bitcoin/issues/18623 +# - cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation artifacts: #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip deploy: off diff --git a/.cirrus.yml b/.cirrus.yml index f4a3878ed8..b839bfa5fb 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,7 +1,7 @@ task: - name: "FreeBsd 12.0 amd64 [GOAL: install] [no depends, only system libs]" + name: "FreeBsd 12.1 amd64 [GOAL: install] [no depends, only system libs]" freebsd_instance: - image: freebsd-12-0-release-amd64 + image_family: freebsd-12-1 # https://cirrus-ci.org/guide/FreeBSD/ cpu: 8 memory: 8G timeout_in: 60m @@ -28,6 +28,21 @@ task: - gmake check ${MAKEJOBS} VERBOSE=1 functional_test_script: - ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast +#task: +# name: "Windows" +# windows_container: +# image: cirrusci/windowsservercore:2019 +# env: +# CIRRUS_SHELL: powershell +# PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%' +# PYTHONUTF8: 1 +# QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' +# QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' +# QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' +# VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' +# VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' +# install_script: +# - choco install python --version=3.7.7 -y task: name: "x86_64 Linux [GOAL: install] [bionic] [Using ./ci/ system]" container: @@ -37,7 +52,7 @@ task: timeout_in: 60m env: MAKEJOBS: "-j9" - RUN_CI_ON_HOST: "1" + DANGER_RUN_CI_ON_HOST: "1" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" diff --git a/.travis.yml b/.travis.yml index 878f5ce518..ea589b4059 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,9 +38,6 @@ cache: - $TRAVIS_BUILD_DIR/depends/sdk-sources - $TRAVIS_BUILD_DIR/ci/scratch/.ccache - $TRAVIS_BUILD_DIR/releases/$HOST - # macOS - - $HOME/Library/Caches/Homebrew - - /usr/local/Homebrew before_cache: - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then brew cleanup; fi stages: @@ -74,7 +71,7 @@ jobs: - stage: lint name: 'lint' env: - cache: false + cache: pip language: python python: '3.5' # Oldest supported version according to doc/dependencies.md install: @@ -91,6 +88,26 @@ jobs: FILE_ENV="./ci/test/00_setup_env_arm.sh" QEMU_USER_CMD="" + - stage: test + name: 's390x native BE [GOAL: install] [bionic] [no depends, no GUI]' + arch: s390x + dist: bionic + addons: + apt: + packages: + - bsdmainutils + - libboost-filesystem-dev + - libboost-system-dev + - libboost-test-dev + - libboost-thread-dev + - libdb++-dev + - libdb-dev + - libevent-dev + env: >- + DANGER_RUN_CI_ON_HOST=true + CI_USE_APT_INSTALL=no + FILE_ENV="./ci/test/00_setup_env_s390x_host.sh" + # s390 build was disabled temporarily because of disk space issues on the Travis VM # # - stage: test @@ -158,6 +175,12 @@ jobs: # Xcode 11.3.1, macOS 10.14, SDK 10.15 # https://docs.travis-ci.com/user/reference/osx/#macos-version osx_image: xcode11.3 + cache: + directories: + - $TRAVIS_BUILD_DIR/ci/scratch/.ccache + - $TRAVIS_BUILD_DIR/releases/$HOST + - $HOME/Library/Caches/Homebrew + - /usr/local/Homebrew addons: homebrew: packages: @@ -171,4 +194,6 @@ jobs: - ccache - zeromq env: >- + DANGER_RUN_CI_ON_HOST=true + CI_USE_APT_INSTALL=no FILE_ENV="./ci/test/00_setup_env_mac_host.sh" diff --git a/build-aux/m4/ax_boost_base.m4 b/build-aux/m4/ax_boost_base.m4 index 16fa69b41f..2ae33f7140 100644 --- a/build-aux/m4/ax_boost_base.m4 +++ b/build-aux/m4/ax_boost_base.m4 @@ -33,7 +33,7 @@ # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 47 +#serial 48 # example boost program (need to pass version) m4_define([_AX_BOOST_BASE_PROGRAM], @@ -123,6 +123,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ dnl are almost assuredly the ones desired. AS_CASE([${host_cpu}], [i?86],[multiarch_libsubdir="lib/i386-${host_os}"], + [armv7l],[multiarch_libsubdir="lib/arm-${host_os}"], [multiarch_libsubdir="lib/${host_cpu}-${host_os}"] ) diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 5f0640ac27..c8b2fb1658 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ #define CLIENT_VERSION_MAJOR 0 /* Minor version */ -#define CLIENT_VERSION_MINOR 19 +#define CLIENT_VERSION_MINOR 20 /* Build revision */ #define CLIENT_VERSION_REVISION 99 diff --git a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in index adf4fa0354..6ec40461c2 100644 --- a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in +++ b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in @@ -8,6 +8,7 @@ <ConfigurationType>StaticLibrary</ConfigurationType> </PropertyGroup> <ItemGroup> + <ClCompile Include="..\..\src\util\url.cpp" /> @SOURCE_FILES@ </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/build_msvc/testconsensus/testconsensus.cpp b/build_msvc/testconsensus/testconsensus.cpp index 571d19957f..5fdd97dc78 100644 --- a/build_msvc/testconsensus/testconsensus.cpp +++ b/build_msvc/testconsensus/testconsensus.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 4c22e4e6c5..e0e555bbd1 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -24,7 +24,7 @@ echo "Fallback to default values in env (if not yet set)" export MAKEJOBS=${MAKEJOBS:--j4} # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) # This folder only exists on the ci host. -export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/} +export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch} # What host to compile for. See also ./depends/README.md # Tests that need cross-compilation export the appropriate HOST. # Tests that run natively guess the host @@ -53,7 +53,6 @@ export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} -export WINEDEBUG=${WINEDEBUG:-fixme-all} export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 2b30b4a5e3..b70a581532 100644 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 6ed6e40bd6..a4dc54d1c1 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index 5753c3af31..67fbd1438a 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,6 @@ export LC_ALL=C.UTF-8 export HOST=x86_64-apple-darwin16 export PIP_PACKAGES="zmq" -export RUN_CI_ON_HOST=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false export GOAL="install" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index d5f39daaf5..28c63f1cf6 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index c8c75dea1a..43ee219ef9 100644 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index eb77dc5b16..c27d525003 100644 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 6bb371920c..9c2be4cfac 100644 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 21c15236d2..87c8b70998 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,6 +10,8 @@ export CONTAINER_NAME=ci_native_qt5 export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash +export RUN_UNIT_TESTS_SEQUENTIAL="true" +export RUN_UNIT_TESTS="false" export GOAL="install" export TEST_PREVIOUS_RELEASES=true export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 4d3f345ca6..73ab5eebb6 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,11 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:16.04 -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang-8 llvm-8 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++" - -# xenial comes with old clang versions that can not parse the sanitizer suppressions files -# Remove unparseable lines as a hacky workaround -sed -i '/^implicit-/d' "${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan" +export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang-8 CXX=clang++-8" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 2a7b32cefc..d5b38a1d88 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,6 +10,10 @@ export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +if [[ "${TRAVIS}" == "true" && "${TRAVIS_REPO_SLUG}" != "bitcoin/bitcoin" ]]; then + export TEST_RUNNER_EXTRA="wallet_disable" # Only run wallet_disable as a smoke test to not hit the 50 min travis time limit +else + export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +fi export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 72ad99fe5d..c180d023de 100644 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_s390x_host.sh b/ci/test/00_setup_env_s390x_host.sh new file mode 100644 index 0000000000..3e31867f2f --- /dev/null +++ b/ci/test/00_setup_env_s390x_host.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=s390x-linux-gnu +export NO_DEPENDS=1 +export BITCOIN_CONFIG="--with-incompatible-bdb --enable-reduce-exports" +export RUN_UNIT_TESTS=true +export RUN_FUNCTIONAL_TESTS=true +export GOAL="install" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index a34933731c..8f0c62a1a6 100644 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index acf7eeb920..32eaaf15bb 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -26,7 +26,7 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order= export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" -env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|TEST_PREVIOUS_RELEASES|PREVIOUS_RELEASES_DIR)' | tee /tmp/env +env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|TEST_PREVIOUS_RELEASES|PREVIOUS_RELEASES_DIR)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN" elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) @@ -35,7 +35,7 @@ fi export P_CI_DIR="$PWD" -if [ -z "$RUN_CI_ON_HOST" ]; then +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" @@ -66,7 +66,7 @@ fi if [[ $DOCKER_NAME_TAG == centos* ]]; then ${CI_RETRY_EXE} DOCKER_EXEC yum -y install epel-release ${CI_RETRY_EXE} DOCKER_EXEC yum -y install $DOCKER_PACKAGES $PACKAGES -elif [ "$TRAVIS_OS_NAME" != "osx" ]; then +elif [ "$CI_USE_APT_INSTALL" != "no" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get update ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES fi @@ -77,6 +77,7 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then else DOCKER_EXEC free -m -h DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) + DOCKER_EXEC echo $(lscpu | grep Endian) DOCKER_EXEC echo "Free disk space:" DOCKER_EXEC df -h fi @@ -90,7 +91,7 @@ export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" -if [ -z "$RUN_CI_ON_HOST" ]; then +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 933f4cea91..17853e75dd 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index b6043cd61e..f568dbe37d 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -1,15 +1,13 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" -if [ -z "$NO_DEPENDS" ]; then - DOCKER_EXEC ccache --max-size=$CCACHE_SIZE -fi +DOCKER_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" BEGIN_FOLD autogen if [ -n "$CONFIG_SHELL" ]; then @@ -44,3 +42,9 @@ trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) END_FOLD + +BEGIN_FOLD cache_stats +DOCKER_EXEC "ccache --version | head -n 1 && ccache --show-stats" +DOCKER_EXEC du -sh "${DEPENDS_DIR}"/*/ +DOCKER_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" +END_FOLD diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 3b32513353..0f03d1770e 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,13 +21,20 @@ if [ -n "$USE_VALGRIND" ]; then END_FOLD fi +bash -c "${CI_WAIT}" & # Print dots in case the tests take a long time to run + if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests - bash -c "${CI_WAIT}" & # Print dots in case the unit tests take a long time to run DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 END_FOLD fi +if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then + BEGIN_FOLD unit-tests-seq + DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_ROOT_DIR}/build/bitcoin-*/src/test/test_bitcoin" --catch_system_errors=no -l test_suite + END_FOLD +fi + if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then BEGIN_FOLD functional-tests DOCKER_EXEC test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index f1d3088748..be7d7fcc1f 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/configure.ac b/configure.ac index a2d33ff5b6..693db0644d 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ([2.69]) define(_CLIENT_VERSION_MAJOR, 0) -define(_CLIENT_VERSION_MINOR, 19) +define(_CLIENT_VERSION_MINOR, 20) define(_CLIENT_VERSION_REVISION, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) @@ -1265,9 +1265,9 @@ if test x$use_pkgconfig = xyes; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - PKG_CHECK_MODULES([EVENT], [libevent],, [AC_MSG_ERROR(libevent not found.)]) + PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR(libevent version 2.0.21 or greater not found.)]) if test x$TARGET_OS != xwindows; then - PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)]) + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR(libevent_pthreads version 2.0.21 or greater not found.)]) fi fi @@ -1285,7 +1285,7 @@ if test x$use_pkgconfig = xyes; then else if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) + AC_CHECK_HEADER([event2/event.h], [use_libevent=yes], AC_MSG_ERROR(libevent headers missing),) AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) if test x$TARGET_OS != xwindows; then AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) @@ -1523,6 +1523,7 @@ AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) +AM_CONDITIONAL([USE_LIBEVENT],[test x$use_libevent = xyes]) AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes]) diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index 6afa4351e7..bc5f09a3e2 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 92120eaff7..084914f11a 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/devtools/previous_release.sh b/contrib/devtools/previous_release.sh index 5ddfdb4e73..b2ecc274fb 100755 --- a/contrib/devtools/previous_release.sh +++ b/contrib/devtools/previous_release.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index c05c38d513..b924698e56 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -206,6 +206,23 @@ def check_MACHO_NX(executable) -> bool: return False return True +def check_MACHO_LAZY_BINDINGS(executable) -> bool: + ''' + Check for no lazy bindings. + We don't use or check for MH_BINDATLOAD. See #18295. + ''' + p = subprocess.Popen([OTOOL_CMD, '-l', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + + for line in stdout.splitlines(): + tokens = line.split() + if 'lazy_bind_off' in tokens or 'lazy_bind_size' in tokens: + if tokens[1] != '0': + return False + return True + CHECKS = { 'ELF': [ ('PIE', check_ELF_PIE), @@ -221,7 +238,8 @@ CHECKS = { 'MACHO': [ ('PIE', check_MACHO_PIE), ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('NX', check_MACHO_NX) + ('NX', check_MACHO_NX), + ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS) ] } diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index e2a8154f16..db738fbbb8 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh index f5cd05a2c3..95b1553215 100755 --- a/contrib/devtools/test_deterministic_coverage.sh +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py index 75034616f7..e005cb96da 100755 --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index a13a42d391..6f79d10e68 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-linux-0.20" +name: "bitcoin-core-linux-0.21" enable_cache: true distro: "ubuntu" suites: @@ -46,7 +46,7 @@ script: | FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS_BASE="-static-libstdc++" + HOST_LDFLAGS_BASE="-static-libstdc++ -Wl,-O2" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 58531c81b4..37f2a534b8 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-osx-0.20" +name: "bitcoin-core-osx-0.21" enable_cache: true distro: "ubuntu" suites: diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 9d96465742..6bcd126662 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -8,6 +8,7 @@ architectures: packages: - "libssl-dev" - "autoconf" +- "automake" - "libtool" - "pkg-config" remotes: diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index c5eea97c77..0cc1adc557 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-core-win-0.20" +name: "bitcoin-core-win-0.21" enable_cache: true distro: "ubuntu" suites: diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index ba3036a89f..0a2c1302c8 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -13,6 +13,7 @@ D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 +82921A4B88FD454B7EB8CE3C796C4109063D4EAF Jon Atack 32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli 4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon C42AFF7C61B3E44A1454CD3557AF762DB3353322 Karl-Johan Alm (kallewoof) @@ -29,6 +30,7 @@ D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille (Location: Leuven, Belgiu 133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost +9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami 79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko 71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 8500379025..9f99b36f88 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -220,8 +220,6 @@ repository and will likely put one up soon. [guix/env-setup]: https://www.gnu.org/software/guix/manual/en/html_node/Build-Environment-Setup.html [guix/substitutes]: https://www.gnu.org/software/guix/manual/en/html_node/Substitutes.html [guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html -[guix/inferiors]: https://www.gnu.org/software/guix/manual/en/html_node/Inferiors.html -[guix/channels]: https://www.gnu.org/software/guix/manual/en/html_node/Channels.html [guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html [debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644 diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 2daa8aba5e..e20b2a048d 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -13,33 +13,106 @@ make -C "${PWD}/depends" -j"$MAX_JOBS" download ${V:+V=1} ${SOURCES_PATH:+SOURCE # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. time-machine() { guix time-machine --url=https://github.com/dongcarl/guix.git \ - --commit=b3a7c72c8b2425f8ddb0fc6e3b1caeed40f86dee \ + --commit=b066c25026f21fb57677aa34692a5034338e7ee3 \ -- "$@" } -# Deterministically build Bitcoin Core for HOSTs (overriable by environment) -for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu}; do +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + output, deploy, and distsrc-* directories before rebuilding + +Hint: To blow everything away, you may want to use: + + $ git clean -xdff --exclude='/depends/SDKs/*' + +Specifically, this will remove all files without an entry in the index, +excluding the SDK directory. Practically speaking, this means that all ignored +and untracked files and directories will be wiped, allowing you to start anew. +EOF +} + +# Deterministically build Bitcoin Core for HOSTs (overridable by environment) +# shellcheck disable=SC2153 +for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu x86_64-w64-mingw32}; do # Display proper warning when the user interrupts the build - trap 'echo "** INT received while building ${host}, you may want to clean up the relevant output and distsrc-* directories before rebuilding"' INT - - # Run the build script 'contrib/guix/libexec/build.sh' in the build - # container specified by 'contrib/guix/manifest.scm' - # shellcheck disable=SC2086 - time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/satoshi/bitcoin/' to + # '/home/satoshi/bitcoin/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/bitcoin + # + # maps our current working directory to /bitcoin + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/bitcoin' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # shellcheck disable=SC2086 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + MAX_JOBS="$MAX_JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + ) done diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index ee207a957c..550b1b8f40 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash export LC_ALL=C set -e -o pipefail +export TZ=UTC # Check that environment variables assumed to be set by the environment are set echo "Building for platform triple ${HOST:?not set} with reference timestamp ${SOURCE_DATE_EPOCH:?not set}..." @@ -36,23 +37,41 @@ store_path() { --expression='s|"[[:space:]]*$||' } -# Determine output paths to use in CROSS_* environment variables -CROSS_GLIBC="$(store_path glibc-cross-${HOST})" -CROSS_GLIBC_STATIC="$(store_path glibc-cross-${HOST} static)" -CROSS_KERNEL="$(store_path linux-libre-headers-cross-${HOST})" -CROSS_GCC="$(store_path gcc-cross-${HOST})" -CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... -CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) - # Set environment variables to point Guix's cross-toolchain to the right # includes/libs for $HOST -# -# NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because -# the limits.h in it is missing a '#include_next <limits.h>' -# -export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" -export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" -export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" +case "$HOST" in + *mingw*) + # Determine output paths to use in CROSS_* environment variables + CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + NATIVE_GCC="$(store_path gcc-glibc-2.27-toolchain)" + export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" + export CPATH="${NATIVE_GCC}/include" + + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + ;; + *linux*) + CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" + CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" + CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIBS=( "${CROSS_GCC}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + # NOTE: CROSS_C_INCLUDE_PATH is missing ${CROSS_GCC_LIB}/include-fixed, because + # the limits.h in it is missing a '#include_next <limits.h>' + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + ;; + *) + exit 1 ;; +esac # Sanity check CROSS_*_PATH directories IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" @@ -74,16 +93,20 @@ export GUIX_LD_WRAPPER_DISABLE_RPATH=yes [ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env # Determine the correct value for -Wl,--dynamic-linker for the current $HOST -glibc_dynamic_linker=$( - case "$HOST" in - i686-linux-gnu) echo /lib/ld-linux.so.2 ;; - x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; - arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; - aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; - riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - *) exit 1 ;; - esac -) +case "$HOST" in + *linux*) + glibc_dynamic_linker=$( + case "$HOST" in + i686-linux-gnu) echo /lib/ld-linux.so.2 ;; + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + *) exit 1 ;; + esac + ) + ;; +esac # Environment variables for determinism export QT_RCC_TEST=1 @@ -136,11 +159,27 @@ DISTNAME="$(basename "$SOURCEDIST" '.tar.gz')" # Binary Tarball Building # ########################### -# Similar flags to Gitian -CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" -HOST_CFLAGS="-O2 -g -ffile-prefix-map=${PWD}=." -HOST_CXXFLAGS="-O2 -g -ffile-prefix-map=${PWD}=." -HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++" +# CONFIGFLAGS +CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" +case "$HOST" in + *linux*) CONFIGFLAGS+=" --enable-glibc-back-compat" ;; +esac + +# CFLAGS +HOST_CFLAGS="-O2 -g" +case "$HOST" in + *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; + *mingw*) HOST_CFLAGS+=" -fno-ident" ;; +esac + +# CXXFLAGS +HOST_CXXFLAGS="$HOST_CFLAGS" + +# LDFLAGS +case "$HOST" in + *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; + *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; +esac # Make $HOST-specific native binaries from depends available in $PATH export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" @@ -160,7 +199,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" ${CONFIGFLAGS} \ CFLAGS="${HOST_CFLAGS}" \ CXXFLAGS="${HOST_CXXFLAGS}" \ - LDFLAGS="${HOST_LDFLAGS}" + ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool @@ -169,9 +208,21 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" # Perform basic ELF security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} - # Check that executables only contain allowed gcc, glibc and libstdc++ - # version symbols for Linux distro back-compatibility. - make -C src --jobs=1 check-symbols ${V:+V=1} + + case "$HOST" in + *linux*|*mingw*) + # Check that executables only contain allowed gcc, glibc and libstdc++ + # version symbols for Linux distro back-compatibility. + make -C src --jobs=1 check-symbols ${V:+V=1} + ;; + esac + + # Make the os-specific installers + case "$HOST" in + *mingw*) + make deploy ${V:+V=1} + ;; + esac # Setup the directory where our Bitcoin Core build for HOST will be # installed. This directory will also later serve as the input for our @@ -180,9 +231,21 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" mkdir -p "${INSTALLPATH}" # Install built Bitcoin Core to $INSTALLPATH make install DESTDIR="${INSTALLPATH}" ${V:+V=1} + + case "$HOST" in + *mingw*) + cp -f --target-directory="$OUTDIR" ./*-setup-unsigned.exe + ;; + esac ( cd installed + case "$HOST" in + *mingw*) + mv --target-directory="$DISTNAME"/lib/ "$DISTNAME"/bin/*.dll + ;; + esac + # Prune libtool and object archives find . -name "lib*.la" -delete find . -name "lib*.a" -delete @@ -196,19 +259,60 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" find "${DISTNAME}/lib" -type f -print0 } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg - cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + case "$HOST" in + *mingw*) + cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" + ;; + *linux*) + cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + ;; + esac # Finally, deterministically produce {non-,}debug binary tarballs ready # for release - find "${DISTNAME}" -not -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) - find "${DISTNAME}" -name "*.dbg" -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + case "$HOST" in + *mingw*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -not -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}.zip" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}" -name "*.dbg" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-w64-mingw32/win64}-debug.zip" && exit 1 ) + ;; + *linux*) + find "${DISTNAME}" -not -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) + find "${DISTNAME}" -name "*.dbg" -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-debug.tar.gz" && exit 1 ) + ;; + esac ) ) + +case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir unsigned + cp --target-directory=unsigned/ "$OUTDIR"/bitcoin-*-setup-unsigned.exe + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + ) + ;; +esac diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index c25ac2977b..86c1a8d27f 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -10,17 +10,34 @@ (gnu packages file) (gnu packages gawk) (gnu packages gcc) + (gnu packages installers) (gnu packages linux) + (gnu packages mingw) (gnu packages perl) (gnu packages pkg-config) (gnu packages python) (gnu packages shells) + (guix build-system gnu) (guix build-system trivial) (guix gexp) (guix packages) (guix profiles) (guix utils)) +(define (make-ssp-fixed-gcc xgcc) + "Given a XGCC package, return a modified package that uses the SSP function +from glibc instead of from libssp.so. Our `symbol-check' script will complain if +we link against libssp.so, and thus will ensure that this works properly. + +Taken from: +http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" + (package + (inherit xgcc) + (arguments + (substitute-keyword-arguments (package-arguments xgcc) + ((#:make-flags flags) + `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) + (define (make-gcc-rpath-link xgcc) "Given a XGCC package, return a modified package that replace each instance of -rpath in the default system spec that's inserted by Guix with -rpath-link" @@ -102,45 +119,77 @@ desirable for building Bitcoin Core release binaries." base-libc base-gcc)) +(define (make-gcc-with-pthreads gcc) + (package-with-extra-configure-variable gcc "--enable-threads" "posix")) + +(define (make-mingw-pthreads-cross-toolchain target) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + (pthreads-xlibc mingw-w64-x86_64-winpthreads) + (pthreads-xgcc (make-gcc-with-pthreads + (cross-gcc target + #:xgcc (make-ssp-fixed-gcc gcc-9) + #:xbinutils xbinutils + #:libc pthreads-xlibc)))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-posix-toolchain")) + (version (package-version pthreads-xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + `(("binutils" ,xbinutils) + ("libc" ,pthreads-xlibc) + ("gcc" ,pthreads-xgcc))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page pthreads-xgcc)) + (license (package-license pthreads-xgcc))))) + + (packages->manifest - (list ;; The Basics - bash-minimal - which - coreutils - util-linux - ;; File(system) inspection - file - grep - diffutils - findutils - ;; File transformation - patch - gawk - sed - ;; Compression and archiving - tar - bzip2 - gzip - xz - zlib - ;; Build tools - gnu-make - libtool - autoconf - automake - pkg-config - ;; Scripting - perl - python-3.7 - ;; Native gcc 9 toolchain targeting glibc 2.27 - (make-gcc-toolchain gcc-9 glibc-2.27) - ;; Cross gcc 9 toolchains targeting glibc 2.27 - (make-bitcoin-cross-toolchain "i686-linux-gnu") - (make-bitcoin-cross-toolchain "x86_64-linux-gnu") - (make-bitcoin-cross-toolchain "aarch64-linux-gnu") - (make-bitcoin-cross-toolchain "arm-linux-gnueabihf") - ;; The glibc 2.27 for riscv64 needs gcc 7 to successfully build (see: - ;; https://www.gnu.org/software/gcc/gcc-7/changes.html#riscv). The final - ;; toolchain is still a gcc 9 toolchain targeting glibc 2.27. - (make-bitcoin-cross-toolchain "riscv64-linux-gnu" - #:base-gcc-for-libc gcc-7))) + (append + (list ;; The Basics + bash-minimal + which + coreutils + util-linux + ;; File(system) inspection + file + grep + diffutils + findutils + ;; File transformation + patch + gawk + sed + ;; Compression and archiving + tar + bzip2 + gzip + xz + zlib + ;; Build tools + gnu-make + libtool + autoconf + automake + pkg-config + ;; Scripting + perl + python-3.7 + ;; Native gcc 9 toolchain targeting glibc 2.27 + (make-gcc-toolchain gcc-9 glibc-2.27)) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + ;; Windows + (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64)) + ((string-contains target "riscv64-linux-") + (list (make-bitcoin-cross-toolchain "riscv64-linux-gnu" + #:base-gcc-for-libc gcc-7))) + ((string-contains target "-linux-") + (list (make-bitcoin-cross-toolchain target))) + (else '()))))) diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index bcca3b7cea..73f54cd488 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -2,7 +2,7 @@ # # linearize-data.py: Construct a linear, no-fork version of the chain. # -# Copyright (c) 2013-2019 The Bitcoin Core developers +# Copyright (c) 2013-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh index 21243ada04..3c7bdf4217 100755 --- a/contrib/macdeploy/extract-osx-sdk.sh +++ b/contrib/macdeploy/extract-osx-sdk.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index c623d5b5e4..e8698994f1 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2013-2019 The Bitcoin Core developers +# Copyright (c) 2013-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh index a26791f0d1..78873dc0c3 100755 --- a/contrib/verify-commits/pre-push-hook.sh +++ b/contrib/verify-commits/pre-push-hook.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/doc/dependencies.md b/doc/dependencies.md index 51a2240107..0cb5311e8b 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -13,7 +13,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | | GCC | | [4.8+](https://gcc.gnu.org/) (C++11 support) | | | | | HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | +| libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | librsvg | | | | | | | MiniUPnPc | [2.0.20180203](https://miniupnp.tuxfamily.org/files) | | No | | | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index da07080724..b33b3ad18a 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -423,27 +423,52 @@ and its `cs_KeyStore` lock for example). Threads ------- -- ThreadScriptCheck : Verifies block scripts. +- [Main thread (`bitcoind`)](https://doxygen.bitcoincore.org/bitcoind_8cpp.html#a0ddf1224851353fc92bfbff6f499fa97) + : Started from `main()` in `bitcoind.cpp`. Responsible for starting up and + shutting down the application. -- ThreadImport : Loads blocks from `blk*.dat` files or `-loadblock=<file>`. +- [ThreadImport (`b-loadblk`)](https://doxygen.bitcoincore.org/init_8cpp.html#ae9e290a0e829ec0198518de2eda579d1) + : Loads blocks from `blk*.dat` files or `-loadblock=<file>` on startup. -- ThreadDNSAddressSeed : Loads addresses of peers from the DNS. +- [ThreadScriptCheck (`b-scriptch.x`)](https://doxygen.bitcoincore.org/validation_8cpp.html#a925a33e7952a157922b0bbb8dab29a20) + : Parallel script validation threads for transactions in blocks. -- ThreadMapPort : Universal plug-and-play startup/shutdown. +- [ThreadHTTP (`b-http`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#abb9f6ea8819672bd9a62d3695070709c) + : Libevent thread to listen for RPC and REST connections. -- ThreadSocketHandler : Sends/Receives data from peers on port 8333. +- [HTTP worker threads(`b-httpworker.x`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#aa6a7bc27265043bc0193220c5ae3a55f) + : Threads to service RPC and REST requests. -- ThreadOpenAddedConnections : Opens network connections to added nodes. +- [Indexer threads (`b-txindex`, etc)](https://doxygen.bitcoincore.org/class_base_index.html#a96a7407421fbf877509248bbe64f8d87) + : One thread per indexer. -- ThreadOpenConnections : Initiates new connections to peers. +- [SchedulerThread (`b-scheduler`)](https://doxygen.bitcoincore.org/class_c_scheduler.html#a14d2800815da93577858ea078aed1fba) + : Does asynchronous background tasks like dumping wallet contents, dumping + addrman and running asynchronous validationinterface callbacks. -- ThreadMessageHandler : Higher-level message handling (sending and receiving). +- [TorControlThread (`b-torcontrol`)](https://doxygen.bitcoincore.org/torcontrol_8cpp.html#a4faed3692d57a0d7bdbecf3b37f72de0) + : Libevent thread for tor connections. -- DumpAddresses : Dumps IP addresses of nodes to `peers.dat`. +- Net threads: -- ThreadRPCServer : Remote procedure call handler, listens on port 8332 for connections and services them. + - [ThreadMessageHandler (`b-msghand`)](https://doxygen.bitcoincore.org/class_c_connman.html#aacdbb7148575a31bb33bc345e2bf22a9) + : Application level message handling (sending and receiving). Almost + all net_processing and validation logic runs on this thread. -- Shutdown : Does an orderly shutdown of everything. + - [ThreadDNSAddressSeed (`b-dnsseed`)](https://doxygen.bitcoincore.org/class_c_connman.html#aa7c6970ed98a4a7bafbc071d24897d13) + : Loads addresses of peers from the DNS. + + - [ThreadMapPort (`b-upnp`)](https://doxygen.bitcoincore.org/net_8cpp.html#a63f82a71c4169290c2db1651a9bbe249) + : Universal plug-and-play startup/shutdown. + + - [ThreadSocketHandler (`b-net`)](https://doxygen.bitcoincore.org/class_c_connman.html#a765597cbfe99c083d8fa3d61bb464e34) + : Sends/Receives data from peers on port 8333. + + - [ThreadOpenAddedConnections (`b-addcon`)](https://doxygen.bitcoincore.org/class_c_connman.html#a0b787caf95e52a346a2b31a580d60a62) + : Opens network connections to added nodes. + + - [ThreadOpenConnections (`b-opencon`)](https://doxygen.bitcoincore.org/class_c_connman.html#a55e9feafc3bab78e5c9d408c207faa45) + : Initiates new connections to peers. Ignoring IDE/editor files -------------------------- @@ -965,7 +990,7 @@ Some good examples of scripted-diff: - [scripted-diff: Rename InitInterfaces to NodeContext](https://github.com/bitcoin/bitcoin/commit/301bd41a2e6765b185bd55f4c541f9e27aeea29d) uses an elegant script to replace occurrences of multiple terms in all source files. -- [scripted-diff: Remove g_connman, g_banman globals](https://github.com/bitcoin/bitcoin/commit/301bd41a2e6765b185bd55f4c541f9e27aeea29d) +- [scripted-diff: Remove g_connman, g_banman globals](https://github.com/bitcoin/bitcoin/commit/8922d7f6b751a3e6b3b9f6fb7961c442877fb65a) replaces specific terms in a list of specific source files. - [scripted-diff: Replace fprintf with tfm::format](https://github.com/bitcoin/bitcoin/commit/fac03ec43a15ad547161e37e53ea82482cc508f9) diff --git a/doc/release-notes.md b/doc/release-notes.md index cd6a4d6b59..0a900ea780 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,5 +1,72 @@ -Please edit the release notes here: +*After branching off for a major version release of Bitcoin Core, use this +template to create the initial release notes draft.* -https://github.com/bitcoin-core/bitcoin-devwiki/wiki/0.20.0-Release-Notes-Draft +*The release notes draft is a temporary file that can be added to by anyone. See +[/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes) +for the process.* + +*Create the draft, named* "*version* Release Notes Draft" +*(e.g. "0.20.0 Release Notes Draft"), as a collaborative wiki in:* + +https://github.com/bitcoin-core/bitcoin-devwiki/wiki/ *Before the final release, move the notes back to this git repository.* + +*version* Release Notes Draft +=============================== + +Bitcoin Core version *version* is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-*version*/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes for older versions), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the datadir needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems using +the Linux kernel, macOS 10.12+, and Windows 7 and newer. It is not recommended +to use Bitcoin Core on unsupported systems. + +Bitcoin Core should also work on most other Unix-like systems but is not +as frequently tested on them. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + +In addition to previously supported CPU platforms, this release's pre-compiled +distribution provides binaries for the RISC-V platform. + +Notable changes +=============== + +Credits +======= + +Thanks to everyone who directly contributed to this release: + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/share/setup.nsi.in b/share/setup.nsi.in index dd9ee54d6d..4b2903a7c9 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -2,6 +2,11 @@ Name "@PACKAGE_NAME@ (64-bit)" RequestExecutionLevel highest SetCompressor /SOLID lzma +SetDateSave off + +# Uncomment these lines when investigating reproducibility errors +#SetCompress off +#SetDatablockOptimize off # General Symbol Definitions !define REGKEY "SOFTWARE\$(^Name)" diff --git a/src/Makefile.am b/src/Makefile.am index 9a4fdfdacf..627df97cad 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -524,9 +524,12 @@ libbitcoin_util_a_SOURCES = \ util/strencodings.cpp \ util/string.cpp \ util/time.cpp \ - util/url.cpp \ $(BITCOIN_CORE_H) +if USE_LIBEVENT +libbitcoin_util_a_SOURCES += util/url.cpp +endif + if GLIBC_BACK_COMPAT libbitcoin_util_a_SOURCES += compat/glibc_compat.cpp AM_LDFLAGS += $(COMPAT_LDFLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 4e88f7004b..059ffd6964 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -3,6 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. FUZZ_TARGETS = \ + test/fuzz/addition_overflow \ test/fuzz/addr_info_deserialize \ test/fuzz/addrdb \ test/fuzz/address_deserialize \ @@ -27,18 +28,22 @@ FUZZ_TARGETS = \ test/fuzz/bloom_filter \ test/fuzz/bloomfilter_deserialize \ test/fuzz/chain \ + test/fuzz/checkqueue \ test/fuzz/coins_deserialize \ + test/fuzz/cuckoocache \ test/fuzz/decode_tx \ test/fuzz/descriptor_parse \ test/fuzz/diskblockindex_deserialize \ test/fuzz/eval_script \ test/fuzz/fee_rate \ test/fuzz/fee_rate_deserialize \ + test/fuzz/fees \ test/fuzz/flat_file_pos_deserialize \ test/fuzz/flatfile \ test/fuzz/float \ test/fuzz/golomb_rice \ test/fuzz/hex \ + test/fuzz/http_request \ test/fuzz/integer \ test/fuzz/inv_deserialize \ test/fuzz/key \ @@ -59,10 +64,12 @@ FUZZ_TARGETS = \ test/fuzz/parse_numbers \ test/fuzz/parse_script \ test/fuzz/parse_univalue \ + test/fuzz/prevector \ test/fuzz/partial_merkle_tree_deserialize \ test/fuzz/partially_signed_transaction_deserialize \ test/fuzz/pow \ test/fuzz/prefilled_transaction_deserialize \ + test/fuzz/process_messages \ test/fuzz/process_message \ test/fuzz/process_message_addr \ test/fuzz/process_message_block \ @@ -195,6 +202,7 @@ BITCOIN_TESTS =\ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ + test/interfaces_tests.cpp \ test/key_io_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ @@ -239,7 +247,9 @@ BITCOIN_TESTS =\ test/uint256_tests.cpp \ test/util_tests.cpp \ test/validation_block_tests.cpp \ + test/validation_chainstatemanager_tests.cpp \ test/validation_flush_tests.cpp \ + test/validationinterface_tests.cpp \ test/versionbits_tests.cpp if ENABLE_WALLET @@ -280,6 +290,12 @@ endif if ENABLE_FUZZ +test_fuzz_addition_overflow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_addition_overflow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_addition_overflow_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_addition_overflow_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addition_overflow_SOURCES = test/fuzz/addition_overflow.cpp + test_fuzz_addr_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDR_INFO_DESERIALIZE=1 test_fuzz_addr_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addr_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -424,12 +440,24 @@ test_fuzz_chain_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_chain_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_chain_SOURCES = test/fuzz/chain.cpp +test_fuzz_checkqueue_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_checkqueue_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_checkqueue_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_checkqueue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_checkqueue_SOURCES = test/fuzz/checkqueue.cpp + test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_coins_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_cuckoocache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_cuckoocache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_cuckoocache_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_cuckoocache_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_cuckoocache_SOURCES = test/fuzz/cuckoocache.cpp + test_fuzz_decode_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_decode_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_decode_tx_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -466,6 +494,12 @@ test_fuzz_fee_rate_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_fee_rate_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_fee_rate_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_fees_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_fees_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_fees_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_fees_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_fees_SOURCES = test/fuzz/fees.cpp + test_fuzz_flat_file_pos_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFLAT_FILE_POS_DESERIALIZE=1 test_fuzz_flat_file_pos_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_flat_file_pos_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -496,6 +530,12 @@ test_fuzz_hex_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_hex_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_hex_SOURCES = test/fuzz/hex.cpp +test_fuzz_http_request_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_http_request_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_http_request_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_http_request_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_http_request_SOURCES = test/fuzz/http_request.cpp + test_fuzz_integer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_integer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_integer_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -616,6 +656,12 @@ test_fuzz_parse_univalue_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_parse_univalue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_parse_univalue_SOURCES = test/fuzz/parse_univalue.cpp +test_fuzz_prevector_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_prevector_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_prevector_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_prevector_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_prevector_SOURCES = test/fuzz/prevector.cpp + test_fuzz_partial_merkle_tree_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIAL_MERKLE_TREE_DESERIALIZE=1 test_fuzz_partial_merkle_tree_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_partial_merkle_tree_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -640,6 +686,12 @@ test_fuzz_prefilled_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_prefilled_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_process_messages_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_process_messages_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_process_messages_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_process_messages_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_messages_SOURCES = test/fuzz/process_messages.cpp + test_fuzz_process_message_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_process_message_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 505d630b7d..d7bc73defb 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -11,6 +11,7 @@ TEST_UTIL_H = \ test/util/blockfilter.h \ test/util/logging.h \ test/util/mining.h \ + test/util/net.h \ test/util/setup_common.h \ test/util/str.h \ test/util/transaction_utils.h \ @@ -22,6 +23,7 @@ libtest_util_a_SOURCES = \ test/util/blockfilter.cpp \ test/util/logging.cpp \ test/util/mining.cpp \ + test/util/net.cpp \ test/util/setup_common.cpp \ test/util/str.cpp \ test/util/transaction_utils.cpp \ diff --git a/src/addrman.cpp b/src/addrman.cpp index 2f8a3a0bd5..02fb2fd491 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index f2fc3999fe..2107840a3a 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,8 +7,8 @@ #include <bech32.h> #include <util/strencodings.h> -#include <vector> #include <string> +#include <vector> static void Bech32Encode(benchmark::State& state) diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 5cf7e43f4b..55a9e0fe42 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/bench.h b/src/bench/bench.h index 6b7a0f76d1..0a0fa99c67 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,11 +5,11 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <chrono> #include <functional> #include <map> #include <string> #include <vector> -#include <chrono> #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 9235d5fe6a..1b75854210 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,39 +17,40 @@ static const char* DEFAULT_PLOT_PLOTLYURL = "https://cdn.plot.ly/plotly-latest.m static const int64_t DEFAULT_PLOT_WIDTH = 1024; static const int64_t DEFAULT_PLOT_HEIGHT = 768; -static void SetupBenchArgs() +static void SetupBenchArgs(ArgsManager& argsman) { - SetupHelpOptions(gArgs); + SetupHelpOptions(argsman); - gArgs.AddArg("-list", "List benchmarks without executing them. Can be combined with -scaling and -filter", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-evals=<n>", strprintf("Number of measurement evaluations to perform. (default: %u)", DEFAULT_BENCH_EVALUATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-scaling=<n>", strprintf("Scaling factor for benchmark's runtime (default: %u)", DEFAULT_BENCH_SCALING), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-printer=(console|plot)", strprintf("Choose printer format. console: print data to console. plot: Print results as HTML graph (default: %s)", DEFAULT_BENCH_PRINTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-plotlyurl=<uri>", strprintf("URL to use for plotly.js (default: %s)", DEFAULT_PLOT_PLOTLYURL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-width=<x>", strprintf("Plot width in pixel (default: %u)", DEFAULT_PLOT_WIDTH), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-height=<x>", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-list", "List benchmarks without executing them. Can be combined with -scaling and -filter", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-evals=<n>", strprintf("Number of measurement evaluations to perform. (default: %u)", DEFAULT_BENCH_EVALUATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-scaling=<n>", strprintf("Scaling factor for benchmark's runtime (default: %u)", DEFAULT_BENCH_SCALING), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-printer=(console|plot)", strprintf("Choose printer format. console: print data to console. plot: Print results as HTML graph (default: %s)", DEFAULT_BENCH_PRINTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-plot-plotlyurl=<uri>", strprintf("URL to use for plotly.js (default: %s)", DEFAULT_PLOT_PLOTLYURL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-plot-width=<x>", strprintf("Plot width in pixel (default: %u)", DEFAULT_PLOT_WIDTH), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-plot-height=<x>", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } int main(int argc, char** argv) { - SetupBenchArgs(); + ArgsManager argsman; + SetupBenchArgs(argsman); std::string error; - if (!gArgs.ParseParameters(argc, argv, error)) { + if (!argsman.ParseParameters(argc, argv, error)) { tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); return EXIT_FAILURE; } - if (HelpRequested(gArgs)) { - std::cout << gArgs.GetHelpMessage(); + if (HelpRequested(argsman)) { + std::cout << argsman.GetHelpMessage(); return EXIT_SUCCESS; } - int64_t evaluations = gArgs.GetArg("-evals", DEFAULT_BENCH_EVALUATIONS); - std::string regex_filter = gArgs.GetArg("-filter", DEFAULT_BENCH_FILTER); - std::string scaling_str = gArgs.GetArg("-scaling", DEFAULT_BENCH_SCALING); - bool is_list_only = gArgs.GetBoolArg("-list", false); + int64_t evaluations = argsman.GetArg("-evals", DEFAULT_BENCH_EVALUATIONS); + std::string regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); + std::string scaling_str = argsman.GetArg("-scaling", DEFAULT_BENCH_SCALING); + bool is_list_only = argsman.GetBoolArg("-list", false); if (evaluations == 0) { return EXIT_SUCCESS; @@ -65,12 +66,12 @@ int main(int argc, char** argv) } std::unique_ptr<benchmark::Printer> printer = MakeUnique<benchmark::ConsolePrinter>(); - std::string printer_arg = gArgs.GetArg("-printer", DEFAULT_BENCH_PRINTER); + std::string printer_arg = argsman.GetArg("-printer", DEFAULT_BENCH_PRINTER); if ("plot" == printer_arg) { printer.reset(new benchmark::PlotlyPrinter( - gArgs.GetArg("-plot-plotlyurl", DEFAULT_PLOT_PLOTLYURL), - gArgs.GetArg("-plot-width", DEFAULT_PLOT_WIDTH), - gArgs.GetArg("-plot-height", DEFAULT_PLOT_HEIGHT))); + argsman.GetArg("-plot-plotlyurl", DEFAULT_PLOT_PLOTLYURL), + argsman.GetArg("-plot-width", DEFAULT_PLOT_WIDTH), + argsman.GetArg("-plot-height", DEFAULT_PLOT_HEIGHT))); } benchmark::BenchRunner::RunAll(*printer, evaluations, scaling_factor, regex_filter, is_list_only); diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index e9dd40293f..fb00189fe1 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp index a02a5315a4..df10f27d03 100644 --- a/src/bench/chacha_poly_aead.cpp +++ b/src/bench/chacha_poly_aead.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <crypto/poly1305.h> // for the POLY1305_TAGLEN constant #include <hash.h> -#include <limits> #include <assert.h> +#include <limits> /* Number of bytes to process per iteration */ static constexpr uint64_t BUFFER_SIZE_TINY = 64; diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 55786126b3..2b2c78905e 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,9 +6,9 @@ #include <bench/data.h> #include <chainparams.h> -#include <validation.h> -#include <streams.h> #include <consensus/validation.h> +#include <streams.h> +#include <validation.h> // These are the two major time-sinks which happen after we have fully received // a block off the wire, but before we can relay the block on to peers using diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index f5f96a0136..6b2c527e52 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -1,14 +1,16 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <util/system.h> #include <checkqueue.h> #include <prevector.h> -#include <vector> -#include <boost/thread/thread.hpp> #include <random.h> +#include <util/system.h> + +#include <boost/thread/thread.hpp> + +#include <vector> static const int MIN_CORES = 2; diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index a9d4d78888..ddcef5121e 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <hash.h> -#include <random.h> -#include <uint256.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> #include <crypto/sha512.h> #include <crypto/siphash.h> +#include <hash.h> +#include <random.h> +#include <uint256.h> /* Number of bytes to hash per iteration */ static const uint64_t BUFFER_SIZE = 1000*1000; diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index a783370b4e..83f279b008 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,6 @@ #include <validation.h> - static void DuplicateInputs(benchmark::State& state) { const CScript SCRIPT_PUB{CScript(OP_TRUE)}; diff --git a/src/bench/examples.cpp b/src/bench/examples.cpp index a2fdab5609..f88150200a 100644 --- a/src/bench/examples.cpp +++ b/src/bench/examples.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/merkle_root.cpp b/src/bench/merkle_root.cpp index bdb2bdbe3d..e84f92feae 100644 --- a/src/bench/merkle_root.cpp +++ b/src/bench/merkle_root.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <uint256.h> -#include <random.h> #include <consensus/merkle.h> +#include <random.h> +#include <uint256.h> static void MerkleRoot(benchmark::State& state) { diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index 2fc6f116a4..511573abac 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> #include <bench/data.h> -#include <validation.h> -#include <streams.h> #include <rpc/blockchain.h> +#include <streams.h> +#include <validation.h> #include <univalue.h> diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 0b34ae3f95..5ed8309b65 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 0381369218..8be0aab1ff 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 6982eaab61..02efefe671 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,6 +15,7 @@ #include <util/strencodings.h> #include <util/system.h> #include <util/translation.h> +#include <util/url.h> #include <functional> #include <memory> @@ -29,6 +30,7 @@ #include <compat/stdin.h> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = urlDecode; static const char DEFAULT_RPCCONNECT[] = "127.0.0.1"; static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; @@ -228,6 +230,7 @@ public: const int ID_NETWORKINFO = 0; const int ID_BLOCKCHAININFO = 1; const int ID_WALLETINFO = 2; + const int ID_BALANCES = 3; /** Create a simulated `getinfo` request. */ UniValue PrepareRequest(const std::string& method, const std::vector<std::string>& args) override @@ -239,6 +242,7 @@ public: result.push_back(JSONRPCRequestObj("getnetworkinfo", NullUniValue, ID_NETWORKINFO)); result.push_back(JSONRPCRequestObj("getblockchaininfo", NullUniValue, ID_BLOCKCHAININFO)); result.push_back(JSONRPCRequestObj("getwalletinfo", NullUniValue, ID_WALLETINFO)); + result.push_back(JSONRPCRequestObj("getbalances", NullUniValue, ID_BALANCES)); return result; } @@ -246,9 +250,9 @@ public: UniValue ProcessReply(const UniValue &batch_in) override { UniValue result(UniValue::VOBJ); - std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in, 3); - // Errors in getnetworkinfo() and getblockchaininfo() are fatal, pass them on - // getwalletinfo() is allowed to fail in case there is no wallet. + std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in, batch_in.size()); + // Errors in getnetworkinfo() and getblockchaininfo() are fatal, pass them on; + // getwalletinfo() and getbalances() are allowed to fail if there is no wallet. if (!batch[ID_NETWORKINFO]["error"].isNull()) { return batch[ID_NETWORKINFO]; } @@ -265,13 +269,15 @@ public: result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); if (!batch[ID_WALLETINFO]["result"].isNull()) { - result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); result.pushKV("keypoolsize", batch[ID_WALLETINFO]["result"]["keypoolsize"]); if (!batch[ID_WALLETINFO]["result"]["unlocked_until"].isNull()) { result.pushKV("unlocked_until", batch[ID_WALLETINFO]["result"]["unlocked_until"]); } result.pushKV("paytxfee", batch[ID_WALLETINFO]["result"]["paytxfee"]); } + if (!batch[ID_BALANCES]["result"].isNull()) { + result.pushKV("balance", batch[ID_BALANCES]["result"]["mine"]["trusted"]); + } result.pushKV("relayfee", batch[ID_NETWORKINFO]["result"]["relayfee"]); result.pushKV("warnings", batch[ID_NETWORKINFO]["result"]["warnings"]); return JSONRPCReplyObj(result, NullUniValue, 1); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index b4b2d7ed52..f54a299a36 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 7f1a4a114b..7f9439788a 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,11 +11,13 @@ #include <logging.h> #include <util/system.h> #include <util/translation.h> +#include <util/url.h> #include <wallet/wallettool.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = nullptr; static void SetupWalletToolArgs() { diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index e284dce0d5..2aa416ca44 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,10 +20,12 @@ #include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> +#include <util/url.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = urlDecode; static void WaitForShutdown(NodeContext& node) { @@ -51,7 +53,7 @@ static bool AppInit(int argc, char* argv[]) // Parameters // // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() - SetupServerArgs(); + SetupServerArgs(node); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { return InitError(strprintf("Error parsing command line arguments: %s\n", error)); diff --git a/src/blockencodings.h b/src/blockencodings.h index be50166cfc..377ac3a1a6 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bloom.cpp b/src/bloom.cpp index bd6069b31f..30af507243 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -102,19 +102,6 @@ bool CBloomFilter::contains(const uint256& hash) const return contains(data); } -void CBloomFilter::clear() -{ - vData.assign(vData.size(),0); - isFull = false; - isEmpty = true; -} - -void CBloomFilter::reset(const unsigned int nNewTweak) -{ - clear(); - nTweak = nNewTweak; -} - bool CBloomFilter::IsWithinSizeConstraints() const { return vData.size() <= MAX_BLOOM_FILTER_SIZE && nHashFuncs <= MAX_HASH_FUNCS; diff --git a/src/bloom.h b/src/bloom.h index 68e76a0258..8e3b7be54d 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -84,9 +84,6 @@ public: bool contains(const COutPoint& outpoint) const; bool contains(const uint256& hash) const; - void clear(); - void reset(const unsigned int nNewTweak); - //! True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS //! (catch a filter which was just deserialized which was too big) bool IsWithinSizeConstraints() const; diff --git a/src/chain.h b/src/chain.h index 64c016a1d6..802e23f775 100644 --- a/src/chain.h +++ b/src/chain.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparams.cpp b/src/chainparams.cpp index f8f2d41138..799474fae2 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparams.h b/src/chainparams.h index 379c75e4be..542ef329da 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/checkqueue.h b/src/checkqueue.h index 9dab5a09ac..e3faa1dec0 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/coins.h b/src/coins.h index ea2e759b54..700570f562 100644 --- a/src/coins.h +++ b/src/coins.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compressor.h b/src/compressor.h index 223603e7e9..478bfff0b6 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index 241cc316a6..af01902c92 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/core_read.cpp b/src/core_read.cpp index d036955641..df78c319ee 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/cuckoocache.h b/src/cuckoocache.h index 4ad5818cdc..2daf676c4a 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_CUCKOOCACHE_H #define BITCOIN_CUCKOOCACHE_H +#include <algorithm> // std::find #include <array> #include <atomic> #include <cmath> diff --git a/src/flatfile.h b/src/flatfile.h index d80682d383..60b3503cc3 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 4d49736140..60c4d06f12 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 11d73b7c9a..176284d103 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -189,7 +189,7 @@ static bool InitHTTPAllowList() } /** HTTP request method as string - use for logging only */ -static std::string RequestMethodString(HTTPRequest::RequestMethod m) +std::string RequestMethodString(HTTPRequest::RequestMethod m) { switch (m) { case HTTPRequest::GET: @@ -510,10 +510,10 @@ void HTTPEvent::trigger(struct timeval* tv) else evtimer_add(ev, tv); // trigger after timeval passed } -HTTPRequest::HTTPRequest(struct evhttp_request* _req) : req(_req), - replySent(false) +HTTPRequest::HTTPRequest(struct evhttp_request* _req, bool _replySent) : req(_req), replySent(_replySent) { } + HTTPRequest::~HTTPRequest() { if (!replySent) { diff --git a/src/httpserver.h b/src/httpserver.h index 46820e6aee..97cd63778a 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -60,7 +60,7 @@ private: bool replySent; public: - explicit HTTPRequest(struct evhttp_request* req); + explicit HTTPRequest(struct evhttp_request* req, bool replySent = false); ~HTTPRequest(); enum RequestMethod { diff --git a/src/index/base.cpp b/src/index/base.cpp index ba71830b6e..7bff463f5b 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/index/base.h b/src/index/base.h index 95d83b9b47..3fab810bb2 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/indirectmap.h b/src/indirectmap.h index 417d500bd4..a9f4b0b9f2 100644 --- a/src/indirectmap.h +++ b/src/indirectmap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/init.cpp b/src/init.cpp index 88caed9ded..de32c0ad71 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -197,7 +197,20 @@ void Shutdown(NodeContext& node) // Because these depend on each-other, we make sure that neither can be // using the other before destroying them. if (node.peer_logic) UnregisterValidationInterface(node.peer_logic.get()); - if (node.connman) node.connman->Stop(); + // Follow the lock order requirements: + // * CheckForStaleTipAndEvictPeers locks cs_main before indirectly calling GetExtraOutboundCount + // which locks cs_vNodes. + // * ProcessMessage locks cs_main and g_cs_orphans before indirectly calling ForEachNode which + // locks cs_vNodes. + // * CConnman::Stop calls DeleteNode, which calls FinalizeNode, which locks cs_main and calls + // EraseOrphansFor, which locks g_cs_orphans. + // + // Thus the implicit locking order requirement is: (1) cs_main, (2) g_cs_orphans, (3) cs_vNodes. + if (node.connman) { + node.connman->StopThreads(); + LOCK2(::cs_main, ::g_cs_orphans); + node.connman->StopNodes(); + } StopTorControl(); @@ -230,13 +243,12 @@ void Shutdown(NodeContext& node) } // FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing - // - // g_chainstate is referenced here directly (instead of ::ChainstateActive()) because it - // may not have been initialized yet. { LOCK(cs_main); - if (g_chainstate && g_chainstate->CanFlushToDisk()) { - g_chainstate->ForceFlushStateToDisk(); + for (CChainState* chainstate : g_chainman.GetAll()) { + if (chainstate->CanFlushToDisk()) { + chainstate->ForceFlushStateToDisk(); + } } } @@ -260,9 +272,11 @@ void Shutdown(NodeContext& node) { LOCK(cs_main); - if (g_chainstate && g_chainstate->CanFlushToDisk()) { - g_chainstate->ForceFlushStateToDisk(); - g_chainstate->ResetCoinsViews(); + for (CChainState* chainstate : g_chainman.GetAll()) { + if (chainstate->CanFlushToDisk()) { + chainstate->ForceFlushStateToDisk(); + chainstate->ResetCoinsViews(); + } } pblocktree.reset(); } @@ -278,20 +292,23 @@ void Shutdown(NodeContext& node) } #endif - try { - if (!fs::remove(GetPidFile())) { - LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); - } - } catch (const fs::filesystem_error& e) { - LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); - } node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); globalVerifyHandle.reset(); ECC_Stop(); + node.args = nullptr; if (node.mempool) node.mempool = nullptr; node.scheduler.reset(); + + try { + if (!fs::remove(GetPidFile())) { + LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); + } + } catch (const fs::filesystem_error& e) { + LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); + } + LogPrintf("%s: done\n", __func__); } @@ -344,8 +361,11 @@ static void OnRPCStopped() LogPrint(BCLog::RPC, "RPC stopped.\n"); } -void SetupServerArgs() +void SetupServerArgs(NodeContext& node) { + assert(!node.args); + node.args = &gArgs; + SetupHelpOptions(gArgs); gArgs.AddArg("-help-debug", "Print help message with debugging options and exit", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); // server-only for now @@ -704,11 +724,17 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) } // scan for better chains in the block chain database, that are not yet connected in the active best chain - BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { - LogPrintf("Failed to connect best block (%s)\n", state.ToString()); - StartShutdown(); - return; + + // We can't hold cs_main during ActivateBestChain even though we're accessing + // the g_chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve + // the relevant pointers before the ABC call. + for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + BlockValidationState state; + if (!chainstate->ActivateBestChain(state, chainparams, nullptr)) { + LogPrintf("Failed to connect best block (%s)\n", state.ToString()); + StartShutdown(); + return; + } } if (gArgs.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { @@ -1498,17 +1524,18 @@ bool AppInitMain(NodeContext& node) bool fLoaded = false; while (!fLoaded && !ShutdownRequested()) { bool fReset = fReindex; + auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); + }; std::string strLoadError; uiInterface.InitMessage(_("Loading block index...").translated); do { const int64_t load_block_index_start_time = GetTimeMillis(); - bool is_coinsview_empty; try { LOCK(cs_main); - // This statement makes ::ChainstateActive() usable. - g_chainstate = MakeUnique<CChainState>(); + g_chainman.InitializeChainstate(); UnloadBlockIndex(); // new CBlockTreeDB tries to delete the existing file, which @@ -1561,43 +1588,53 @@ bool AppInitMain(NodeContext& node) // At this point we're either in reindex or we've loaded a useful // block tree into BlockIndex()! - ::ChainstateActive().InitCoinsDB( - /* cache_size_bytes */ nCoinDBCache, - /* in_memory */ false, - /* should_wipe */ fReset || fReindexChainState); - - ::ChainstateActive().CoinsErrorCatcher().AddReadErrCallback([]() { - uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down.").translated, - "", CClientUIInterface::MSG_ERROR); - }); - - // If necessary, upgrade from older database format. - // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!::ChainstateActive().CoinsDB().Upgrade()) { - strLoadError = _("Error upgrading chainstate database").translated; - break; - } - - // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!::ChainstateActive().ReplayBlocks(chainparams)) { - strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; - break; - } - - // The on-disk coinsdb is now in a good state, create the cache - ::ChainstateActive().InitCoinsCache(); - assert(::ChainstateActive().CanFlushToDisk()); + bool failed_chainstate_init = false; + + for (CChainState* chainstate : g_chainman.GetAll()) { + LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); + chainstate->InitCoinsDB( + /* cache_size_bytes */ nCoinDBCache, + /* in_memory */ false, + /* should_wipe */ fReset || fReindexChainState); + + chainstate->CoinsErrorCatcher().AddReadErrCallback([]() { + uiInterface.ThreadSafeMessageBox( + _("Error reading from database, shutting down.").translated, + "", CClientUIInterface::MSG_ERROR); + }); + + // If necessary, upgrade from older database format. + // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate + if (!chainstate->CoinsDB().Upgrade()) { + strLoadError = _("Error upgrading chainstate database").translated; + failed_chainstate_init = true; + break; + } - is_coinsview_empty = fReset || fReindexChainState || - ::ChainstateActive().CoinsTip().GetBestBlock().IsNull(); - if (!is_coinsview_empty) { - // LoadChainTip initializes the chain based on CoinsTip()'s best block - if (!::ChainstateActive().LoadChainTip(chainparams)) { - strLoadError = _("Error initializing block database").translated; + // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate + if (!chainstate->ReplayBlocks(chainparams)) { + strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; + failed_chainstate_init = true; break; } - assert(::ChainActive().Tip() != nullptr); + + // The on-disk coinsdb is now in a good state, create the cache + chainstate->InitCoinsCache(); + assert(chainstate->CanFlushToDisk()); + + if (!is_coinsview_empty(chainstate)) { + // LoadChainTip initializes the chain based on CoinsTip()'s best block + if (!chainstate->LoadChainTip(chainparams)) { + strLoadError = _("Error initializing block database").translated; + failed_chainstate_init = true; + break; // out of the per-chainstate loop + } + assert(chainstate->m_chain.Tip() != nullptr); + } + } + + if (failed_chainstate_init) { + break; // out of the chainstate activation do-while } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); @@ -1605,49 +1642,76 @@ bool AppInitMain(NodeContext& node) break; } - if (!fReset) { - // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. - // It both disconnects blocks based on ::ChainActive(), and drops block data in - // BlockIndex() based on lack of available witness data. - uiInterface.InitMessage(_("Rewinding blocks...").translated); - if (!RewindBlockIndex(chainparams)) { - strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain").translated; - break; + bool failed_rewind{false}; + // Can't hold cs_main while calling RewindBlockIndex, so retrieve the relevant + // chainstates beforehand. + for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + if (!fReset) { + // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. + // It both disconnects blocks based on the chainstate, and drops block data in + // BlockIndex() based on lack of available witness data. + uiInterface.InitMessage(_("Rewinding blocks...").translated); + if (!chainstate->RewindBlockIndex(chainparams)) { + strLoadError = _( + "Unable to rewind the database to a pre-fork state. " + "You will need to redownload the blockchain").translated; + failed_rewind = true; + break; // out of the per-chainstate loop + } } } + if (failed_rewind) { + break; // out of the chainstate activation do-while + } + + bool failed_verification = false; + try { LOCK(cs_main); - if (!is_coinsview_empty) { - uiInterface.InitMessage(_("Verifying blocks...").translated); - if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { - LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", - MIN_BLOCKS_TO_KEEP); - } - CBlockIndex* tip = ::ChainActive().Tip(); - RPCNotifyBlockChange(true, tip); - if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { - strLoadError = _("The block database contains a block which appears to be from the future. " - "This may be due to your computer's date and time being set incorrectly. " - "Only rebuild the block database if you are sure that your computer's date and time are correct").translated; - break; - } - - if (!CVerifyDB().VerifyDB(chainparams, &::ChainstateActive().CoinsDB(), gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), - gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { - strLoadError = _("Corrupted block database detected").translated; - break; + for (CChainState* chainstate : g_chainman.GetAll()) { + if (!is_coinsview_empty(chainstate)) { + uiInterface.InitMessage(_("Verifying blocks...").translated); + if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { + LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", + MIN_BLOCKS_TO_KEEP); + } + + const CBlockIndex* tip = chainstate->m_chain.Tip(); + RPCNotifyBlockChange(true, tip); + if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { + strLoadError = _("The block database contains a block which appears to be from the future. " + "This may be due to your computer's date and time being set incorrectly. " + "Only rebuild the block database if you are sure that your computer's date and time are correct").translated; + failed_verification = true; + break; + } + + // Only verify the DB of the active chainstate. This is fixed in later + // work when we allow VerifyDB to be parameterized by chainstate. + if (&::ChainstateActive() == chainstate && + !CVerifyDB().VerifyDB( + chainparams, &chainstate->CoinsDB(), + gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), + gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { + strLoadError = _("Corrupted block database detected").translated; + failed_verification = true; + break; + } } } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); strLoadError = _("Error opening block database").translated; + failed_verification = true; break; } - fLoaded = true; - LogPrintf(" block index %15dms\n", GetTimeMillis() - load_block_index_start_time); + if (!failed_verification) { + fLoaded = true; + LogPrintf(" block index %15dms\n", GetTimeMillis() - load_block_index_start_time); + } } while(false); if (!fLoaded && !ShutdownRequested()) { @@ -1711,8 +1775,11 @@ bool AppInitMain(NodeContext& node) LogPrintf("Unsetting NODE_NETWORK on prune mode\n"); nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK); if (!fReindex) { - uiInterface.InitMessage(_("Pruning blockstore...").translated); - ::ChainstateActive().PruneAndFlush(); + LOCK(cs_main); + for (CChainState* chainstate : g_chainman.GetAll()) { + uiInterface.InitMessage(_("Pruning blockstore...").translated); + chainstate->PruneAndFlush(); + } } } diff --git a/src/init.h b/src/init.h index f74ae5a47a..ef568b6f38 100644 --- a/src/init.h +++ b/src/init.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -54,9 +54,9 @@ bool AppInitLockDataDirectory(); bool AppInitMain(NodeContext& node); /** - * Setup the arguments for gArgs + * Register all arguments with the ArgsManager */ -void SetupServerArgs(); +void SetupServerArgs(NodeContext& node); /** Returns licensing information (for -version) */ std::string LicenseInfo(); diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index 0b3cd08e22..0e7641ae32 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -38,6 +38,21 @@ namespace interfaces { namespace { +bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock) +{ + if (!index) return false; + if (block.m_hash) *block.m_hash = index->GetBlockHash(); + if (block.m_height) *block.m_height = index->nHeight; + if (block.m_time) *block.m_time = index->GetBlockTime(); + if (block.m_max_time) *block.m_max_time = index->GetBlockTimeMax(); + if (block.m_mtp_time) *block.m_mtp_time = index->GetMedianTimePast(); + if (block.m_data) { + REVERSE_LOCK(lock); + if (!ReadBlockFromDisk(*block.m_data, index, Params().GetConsensus())) block.m_data->SetNull(); + } + return true; +} + class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> { Optional<int> getHeight() override @@ -65,20 +80,6 @@ class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> assert(block != nullptr); return block->GetBlockHash(); } - int64_t getBlockTime(int height) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; - assert(block != nullptr); - return block->GetBlockTime(); - } - int64_t getBlockMedianTimePast(int height) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; - assert(block != nullptr); - return block->GetMedianTimePast(); - } bool haveBlockOnDisk(int height) override { LockAssertion lock(::cs_main); @@ -95,20 +96,6 @@ class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> } return nullopt; } - Optional<int> findPruned(int start_height, Optional<int> stop_height) override - { - LockAssertion lock(::cs_main); - if (::fPruneMode) { - CBlockIndex* block = stop_height ? ::ChainActive()[*stop_height] : ::ChainActive().Tip(); - while (block && block->nHeight >= start_height) { - if ((block->nStatus & BLOCK_HAVE_DATA) == 0) { - return block->nHeight; - } - block = block->pprev; - } - } - return nullopt; - } Optional<int> findFork(const uint256& hash, Optional<int>* height) override { LockAssertion lock(::cs_main); @@ -247,26 +234,50 @@ public: std::unique_ptr<Chain::Lock> result = std::move(lock); // Temporary to avoid CWG 1579 return result; } - bool findBlock(const uint256& hash, CBlock* block, int64_t* time, int64_t* time_max) override + bool findBlock(const uint256& hash, const FoundBlock& block) override { - CBlockIndex* index; - { - LOCK(cs_main); - index = LookupBlockIndex(hash); - if (!index) { - return false; - } - if (time) { - *time = index->GetBlockTime(); - } - if (time_max) { - *time_max = index->GetBlockTimeMax(); + WAIT_LOCK(cs_main, lock); + return FillBlock(LookupBlockIndex(hash), block, lock); + } + bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override + { + WAIT_LOCK(cs_main, lock); + return FillBlock(ChainActive().FindEarliestAtLeast(min_time, min_height), block, lock); + } + bool findNextBlock(const uint256& block_hash, int block_height, const FoundBlock& next, bool* reorg) override { + WAIT_LOCK(cs_main, lock); + CBlockIndex* block = ChainActive()[block_height]; + if (block && block->GetBlockHash() != block_hash) block = nullptr; + if (reorg) *reorg = !block; + return FillBlock(block ? ChainActive()[block_height + 1] : nullptr, next, lock); + } + bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out) override + { + WAIT_LOCK(cs_main, lock); + if (const CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { + return FillBlock(ancestor, ancestor_out, lock); } } - if (block && !ReadBlockFromDisk(*block, index, Params().GetConsensus())) { - block->SetNull(); - } - return true; + return FillBlock(nullptr, ancestor_out, lock); + } + bool findAncestorByHash(const uint256& block_hash, const uint256& ancestor_hash, const FoundBlock& ancestor_out) override + { + WAIT_LOCK(cs_main, lock); + const CBlockIndex* block = LookupBlockIndex(block_hash); + const CBlockIndex* ancestor = LookupBlockIndex(ancestor_hash); + if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; + return FillBlock(ancestor, ancestor_out, lock); + } + bool findCommonAncestor(const uint256& block_hash1, const uint256& block_hash2, const FoundBlock& ancestor_out, const FoundBlock& block1_out, const FoundBlock& block2_out) override + { + WAIT_LOCK(cs_main, lock); + const CBlockIndex* block1 = LookupBlockIndex(block_hash1); + const CBlockIndex* block2 = LookupBlockIndex(block_hash2); + const CBlockIndex* ancestor = block1 && block2 ? LastCommonAncestor(block1, block2) : nullptr; + // Using & instead of && below to avoid short circuiting and leaving + // output uninitialized. + return FillBlock(ancestor, ancestor_out, lock) & FillBlock(block1, block1_out, lock) & FillBlock(block2, block2_out, lock); } void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override @@ -274,6 +285,25 @@ public: LOCK(cs_main); return GuessVerificationProgress(Params().TxData(), LookupBlockIndex(block_hash)); } + bool hasBlocks(const uint256& block_hash, int min_height, Optional<int> max_height) override + { + // hasBlocks returns true if all ancestors of block_hash in specified + // range have block data (are not pruned), false if any ancestors in + // specified range are missing data. + // + // For simplicity and robustness, min_height and max_height are only + // used to limit the range, and passing min_height that's too low or + // max_height that's too high will not crash or change the result. + LOCK(::cs_main); + if (CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (max_height && block->nHeight >= *max_height) block = block->GetAncestor(*max_height); + for (; block->nStatus & BLOCK_HAVE_DATA; block = block->pprev) { + // Check pprev to not segfault if min_height is too low + if (block->nHeight <= min_height || !block->pprev) return true; + } + } + return false; + } RBFTransactionState isRBFOptIn(const CTransaction& tx) override { LOCK(::mempool.cs); diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index e1bc9bbbf3..fb77c81cdc 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -30,6 +30,27 @@ namespace interfaces { class Handler; class Wallet; +//! Helper for findBlock to selectively return pieces of block data. +class FoundBlock +{ +public: + FoundBlock& hash(uint256& hash) { m_hash = &hash; return *this; } + FoundBlock& height(int& height) { m_height = &height; return *this; } + FoundBlock& time(int64_t& time) { m_time = &time; return *this; } + FoundBlock& maxTime(int64_t& max_time) { m_max_time = &max_time; return *this; } + FoundBlock& mtpTime(int64_t& mtp_time) { m_mtp_time = &mtp_time; return *this; } + //! Read block data from disk. If the block exists but doesn't have data + //! (for example due to pruning), the CBlock variable will be set to null. + FoundBlock& data(CBlock& data) { m_data = &data; return *this; } + + uint256* m_hash = nullptr; + int* m_height = nullptr; + int64_t* m_time = nullptr; + int64_t* m_max_time = nullptr; + int64_t* m_mtp_time = nullptr; + CBlock* m_data = nullptr; +}; + //! Interface giving clients (wallet processes, maybe other analysis tools in //! the future) ability to access to the chain state, receive notifications, //! estimate fees, and submit transactions. @@ -79,13 +100,6 @@ public: //! Get block hash. Height must be valid or this function will abort. virtual uint256 getBlockHash(int height) = 0; - //! Get block time. Height must be valid or this function will abort. - virtual int64_t getBlockTime(int height) = 0; - - //! Get block median time past. Height must be valid or this function - //! will abort. - virtual int64_t getBlockMedianTimePast(int height) = 0; - //! Check that the block is available on disk (i.e. has not been //! pruned), and contains transactions. virtual bool haveBlockOnDisk(int height) = 0; @@ -97,10 +111,6 @@ public: //! (to avoid the cost of a second lookup in case this information is needed.) virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) = 0; - //! Return height of last block in the specified range which is pruned, or - //! nullopt if no block in the range is pruned. Range is inclusive. - virtual Optional<int> findPruned(int start_height = 0, Optional<int> stop_height = nullopt) = 0; - //! Return height of the specified block if it is on the chain, otherwise //! return the height of the highest block on chain that's an ancestor //! of the specified block, or nullopt if there is no common ancestor. @@ -127,14 +137,36 @@ public: //! Return whether node has the block and optionally return block metadata //! or contents. - //! - //! If a block pointer is provided to retrieve the block contents, and the - //! block exists but doesn't have data (for example due to pruning), the - //! block will be empty and all fields set to null. - virtual bool findBlock(const uint256& hash, - CBlock* block = nullptr, - int64_t* time = nullptr, - int64_t* max_time = nullptr) = 0; + virtual bool findBlock(const uint256& hash, const FoundBlock& block={}) = 0; + + //! Find first block in the chain with timestamp >= the given time + //! and height >= than the given height, return false if there is no block + //! with a high enough timestamp and height. Optionally return block + //! information. + virtual bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block={}) = 0; + + //! Find next block if block is part of current chain. Also flag if + //! there was a reorg and the specified block hash is no longer in the + //! current chain, and optionally return block information. + virtual bool findNextBlock(const uint256& block_hash, int block_height, const FoundBlock& next={}, bool* reorg=nullptr) = 0; + + //! Find ancestor of block at specified height and optionally return + //! ancestor information. + virtual bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out={}) = 0; + + //! Return whether block descends from a specified ancestor, and + //! optionally return ancestor information. + virtual bool findAncestorByHash(const uint256& block_hash, + const uint256& ancestor_hash, + const FoundBlock& ancestor_out={}) = 0; + + //! Find most recent common ancestor between two blocks and optionally + //! return block information. + virtual bool findCommonAncestor(const uint256& block_hash1, + const uint256& block_hash2, + const FoundBlock& ancestor_out={}, + const FoundBlock& block1_out={}, + const FoundBlock& block2_out={}) = 0; //! Look up unspent output information. Returns coins in the mempool and in //! the current chain UTXO set. Iterates through all the keys in the map and @@ -145,6 +177,11 @@ public: //! the specified block hash are verified. virtual double guessVerificationProgress(const uint256& block_hash) = 0; + //! Return true if data is available for all blocks in the specified range + //! of blocks. This checks all blocks that are ancestors of block_hash in + //! the height range from min_height to max_height, inclusive. + virtual bool hasBlocks(const uint256& block_hash, int min_height = 0, Optional<int> max_height = {}) = 0; + //! Check if transaction is RBF opt in. virtual RBFTransactionState isRBFOptIn(const CTransaction& tx) = 0; @@ -283,6 +320,12 @@ public: //! Shut down client. virtual void stop() = 0; + + //! Set mock time. + virtual void setMockTime(int64_t time) = 0; + + //! Return interfaces for accessing wallets (if any). + virtual std::vector<std::unique_ptr<Wallet>> getWallets() = 0; }; //! Return implementation of Chain interface. diff --git a/src/interfaces/handler.cpp b/src/interfaces/handler.cpp index 95035c1b54..4134a4527f 100644 --- a/src/interfaces/handler.cpp +++ b/src/interfaces/handler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/interfaces/handler.h b/src/interfaces/handler.h index fbac3c6b71..11baf9dd65 100644 --- a/src/interfaces/handler.h +++ b/src/interfaces/handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 905173d20b..5ebbd61584 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -97,7 +97,7 @@ public: StopMapPort(); } } - void setupServerArgs() override { return SetupServerArgs(); } + void setupServerArgs() override { return SetupServerArgs(m_context); } bool getProxy(Network net, proxyType& proxy_info) override { return GetProxy(net, proxy_info); } size_t getNodeCount(CConnman::NumConnections flags) override { @@ -253,8 +253,9 @@ public: std::vector<std::unique_ptr<Wallet>> getWallets() override { std::vector<std::unique_ptr<Wallet>> wallets; - for (const std::shared_ptr<CWallet>& wallet : GetWallets()) { - wallets.emplace_back(MakeWallet(wallet)); + for (auto& client : m_context.chain_clients) { + auto client_wallets = client->getWallets(); + std::move(client_wallets.begin(), client_wallets.end(), std::back_inserter(wallets)); } return wallets; } diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 645a8709d2..942952d9c6 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,7 @@ #include <sync.h> #include <ui_interface.h> #include <uint256.h> +#include <util/check.h> #include <util/system.h> #include <wallet/feebumper.h> #include <wallet/fees.h> @@ -62,7 +63,7 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx) { WalletTxStatus result; - result.block_height = locked_chain.getBlockHeight(wtx.m_confirm.hashBlock).get_value_or(std::numeric_limits<int>::max()); + result.block_height = wtx.m_confirm.block_height > 0 ? wtx.m_confirm.block_height : std::numeric_limits<int>::max(); result.blocks_to_maturity = wtx.GetBlocksToMaturity(); result.depth_in_main_chain = wtx.GetDepthInMainChain(); result.time_received = wtx.nTimeReceived; @@ -151,12 +152,12 @@ public: std::string* purpose) override { LOCK(m_wallet->cs_wallet); - auto it = m_wallet->mapAddressBook.find(dest); - if (it == m_wallet->mapAddressBook.end()) { + auto it = m_wallet->m_address_book.find(dest); + if (it == m_wallet->m_address_book.end() || it->second.IsChange()) { return false; } if (name) { - *name = it->second.name; + *name = it->second.GetLabel(); } if (is_mine) { *is_mine = m_wallet->IsMine(dest); @@ -170,8 +171,9 @@ public: { LOCK(m_wallet->cs_wallet); std::vector<WalletAddress> result; - for (const auto& item : m_wallet->mapAddressBook) { - result.emplace_back(item.first, m_wallet->IsMine(item.first), item.second.name, item.second.purpose); + for (const auto& item : m_wallet->m_address_book) { + if (item.second.IsChange()) continue; + result.emplace_back(item.first, m_wallet->IsMine(item.first), item.second.GetLabel(), item.second.purpose); } return result; } @@ -317,13 +319,9 @@ public: if (mi == m_wallet->mapWallet.end()) { return false; } - if (Optional<int> height = locked_chain->getHeight()) { - num_blocks = *height; - block_time = locked_chain->getBlockTime(*height); - } else { - num_blocks = -1; - block_time = -1; - } + num_blocks = m_wallet->GetLastBlockHeight(); + block_time = -1; + CHECK_NONFATAL(m_wallet->chain().findBlock(m_wallet->GetLastBlockHash(), FoundBlock().time(block_time))); tx_status = MakeWalletTxStatus(*locked_chain, mi->second); return true; } @@ -372,12 +370,12 @@ public: { auto locked_chain = m_wallet->chain().lock(true /* try_lock */); if (!locked_chain) return false; - num_blocks = locked_chain->getHeight().get_value_or(-1); - if (!force && num_blocks == cached_num_blocks) return false; TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; } + num_blocks = m_wallet->GetLastBlockHeight(); + if (!force && num_blocks == cached_num_blocks) return false; balances = getBalances(); return true; } @@ -518,6 +516,15 @@ public: void start(CScheduler& scheduler) override { return StartWallets(scheduler); } void flush() override { return FlushWallets(); } void stop() override { return StopWallets(); } + void setMockTime(int64_t time) override { return SetMockTime(time); } + std::vector<std::unique_ptr<Wallet>> getWallets() override + { + std::vector<std::unique_ptr<Wallet>> wallets; + for (const auto& wallet : GetWallets()) { + wallets.emplace_back(MakeWallet(wallet)); + } + return wallets; + } ~WalletClientImpl() override { UnloadWallets(); } Chain& m_chain; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index dbb0912230..0157e9cf95 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,6 @@ #include <functional> #include <map> #include <memory> -#include <psbt.h> #include <stdint.h> #include <string> #include <tuple> @@ -26,12 +25,13 @@ class CCoinControl; class CFeeRate; class CKey; class CWallet; -enum isminetype : unsigned int; enum class FeeReason; -typedef uint8_t isminefilter; - enum class OutputType; +enum class TransactionError; +enum isminetype : unsigned int; struct CRecipient; +struct PartiallySignedTransaction; +typedef uint8_t isminefilter; namespace interfaces { diff --git a/src/logging/timer.h b/src/logging/timer.h index 2b27c71080..21bb3d121e 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/memusage.h b/src/memusage.h index 24eb450465..a6e894129a 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/miner.cpp b/src/miner.cpp index 61d27d17c1..d9dcbe8a70 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -39,6 +39,17 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam return nNewTime - nOldTime; } +void RegenerateCommitments(CBlock& block) +{ + CMutableTransaction tx{*block.vtx.at(0)}; + tx.vout.erase(tx.vout.begin() + GetWitnessCommitmentIndex(block)); + block.vtx.at(0) = MakeTransactionRef(tx); + + GenerateCoinbaseCommitment(block, WITH_LOCK(cs_main, return LookupBlockIndex(block.hashPrevBlock)), Params().GetConsensus()); + + block.hashMerkleRoot = BlockMerkleRoot(block); +} + BlockAssembler::Options::Options() { blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; diff --git a/src/miner.h b/src/miner.h index cc8fc31a9f..69296f9078 100644 --- a/src/miner.h +++ b/src/miner.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -203,4 +203,7 @@ private: void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce); int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); +/** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ +void RegenerateCommitments(CBlock& block); + #endif // BITCOIN_MINER_H diff --git a/src/net.cpp b/src/net.cpp index 8352c40b98..dcc613ba88 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2387,7 +2387,7 @@ void CConnman::Interrupt() } } -void CConnman::Stop() +void CConnman::StopThreads() { if (threadMessageHandler.joinable()) threadMessageHandler.join(); @@ -2399,14 +2399,17 @@ void CConnman::Stop() threadDNSAddressSeed.join(); if (threadSocketHandler.joinable()) threadSocketHandler.join(); +} - if (fAddressesInitialized) - { +void CConnman::StopNodes() +{ + if (fAddressesInitialized) { DumpAddresses(); fAddressesInitialized = false; } // Close sockets + LOCK(cs_vNodes); for (CNode* pnode : vNodes) pnode->CloseSocketDisconnect(); for (ListenSocket& hListenSocket : vhListenSocket) @@ -2415,10 +2418,10 @@ void CConnman::Stop() LogPrintf("CloseSocket(hListenSocket) failed with error %s\n", NetworkErrorString(WSAGetLastError())); // clean up some globals (to help leak detection) - for (CNode *pnode : vNodes) { + for (CNode* pnode : vNodes) { DeleteNode(pnode); } - for (CNode *pnode : vNodesDisconnected) { + for (CNode* pnode : vNodesDisconnected) { DeleteNode(pnode); } vNodes.clear(); @@ -2433,7 +2436,7 @@ void CConnman::DeleteNode(CNode* pnode) assert(pnode); bool fUpdateConnectionTime = false; m_msgproc->FinalizeNode(pnode->GetId(), fUpdateConnectionTime); - if(fUpdateConnectionTime) { + if (fUpdateConnectionTime) { addrman.Connected(pnode->addr); } delete pnode; @@ -188,16 +188,13 @@ public: ~CConnman(); bool Start(CScheduler& scheduler, const Options& options); - // TODO: Remove NO_THREAD_SAFETY_ANALYSIS. Lock cs_vNodes before reading the variable vNodes. - // - // When removing NO_THREAD_SAFETY_ANALYSIS be aware of the following lock order requirements: - // * CheckForStaleTipAndEvictPeers locks cs_main before indirectly calling GetExtraOutboundCount - // which locks cs_vNodes. - // * ProcessMessage locks cs_main and g_cs_orphans before indirectly calling ForEachNode which - // locks cs_vNodes. - // - // Thus the implicit locking order requirement is: (1) cs_main, (2) g_cs_orphans, (3) cs_vNodes. - void Stop() NO_THREAD_SAFETY_ANALYSIS; + void StopThreads(); + void StopNodes(); + void Stop() + { + StopThreads(); + StopNodes(); + }; void Interrupt(); bool GetNetworkActive() const { return fNetworkActive; }; @@ -482,6 +479,7 @@ private: std::atomic<int64_t> m_next_send_inv_to_incoming{0}; friend struct CConnmanTest; + friend struct ConnmanTestMsg; }; void Discover(); void StartMapPort(); @@ -721,6 +719,8 @@ public: class CNode { friend class CConnman; + friend struct ConnmanTestMsg; + public: std::unique_ptr<TransportDeserializer> m_deserializer; std::unique_ptr<TransportSerializer> m_serializer; @@ -797,8 +797,8 @@ public: std::vector<CAddress> vAddrToSend; const std::unique_ptr<CRollingBloomFilter> m_addr_known; bool fGetAddr{false}; - int64_t nNextAddrSend GUARDED_BY(cs_sendProcessing){0}; - int64_t nNextLocalAddrSend GUARDED_BY(cs_sendProcessing){0}; + std::chrono::microseconds m_next_addr_send GUARDED_BY(cs_sendProcessing){0}; + std::chrono::microseconds m_next_local_addr_send GUARDED_BY(cs_sendProcessing){0}; bool IsAddrRelayPeer() const { return m_addr_known != nullptr; } @@ -809,14 +809,13 @@ public: RecursiveMutex cs_inventory; struct TxRelay { - TxRelay() { pfilter = MakeUnique<CBloomFilter>(); } mutable RecursiveMutex cs_filter; // We use fRelayTxes for two purposes - // a) it allows us to not relay tx invs before receiving the peer's version message // b) the peer may tell us in its version message that we should not relay tx invs // unless it loads a bloom filter. bool fRelayTxes GUARDED_BY(cs_filter){false}; - std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter); + std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter){nullptr}; mutable RecursiveMutex cs_tx_inventory; CRollingBloomFilter filterInventoryKnown GUARDED_BY(cs_tx_inventory){50000, 0.000001}; diff --git a/src/net_permissions.h b/src/net_permissions.h index ad74848347..962a2159fc 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/net_processing.cpp b/src/net_processing.cpp index ab430cbe19..96e9178b6e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -97,10 +97,10 @@ void EraseOrphansFor(NodeId peer); /** Increase a node's misbehavior score. */ void Misbehaving(NodeId nodeid, int howmuch, const std::string& message="") EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Average delay between local address broadcasts in seconds. */ -static constexpr unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 60 * 60; -/** Average delay between peer address broadcasts in seconds. */ -static const unsigned int AVG_ADDRESS_BROADCAST_INTERVAL = 30; +/** Average delay between local address broadcasts */ +static constexpr std::chrono::hours AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24}; +/** Average delay between peer address broadcasts */ +static constexpr std::chrono::seconds AVG_ADDRESS_BROADCAST_INTERVAL{30}; /** Average delay between trickled inventory transmissions in seconds. * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; @@ -1365,7 +1365,7 @@ void RelayTransaction(const uint256& txid, const CConnman& connman) }); } -static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connman) +static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& connman) { unsigned int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) @@ -1373,7 +1373,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connma // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the m_addr_knowns of the chosen nodes prevent repeats uint64_t hashAddr = addr.GetHash(); - const CSipHasher hasher = connman->GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); + const CSipHasher hasher = connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24 * 60 * 60)); FastRandomContext insecure_rand; std::array<std::pair<uint64_t, CNode*>,2> best{{{0, nullptr}, {0, nullptr}}}; @@ -1398,7 +1398,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connma } }; - connman->ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); + connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); } void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, const CInv& inv, CConnman* connman) @@ -1911,9 +1911,9 @@ void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uin } } -bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) +bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) { - LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); + LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom->GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -1922,8 +1922,8 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR if (!(pfrom->GetLocalServices() & NODE_BLOOM) && - (strCommand == NetMsgType::FILTERLOAD || - strCommand == NetMsgType::FILTERADD)) + (msg_type == NetMsgType::FILTERLOAD || + msg_type == NetMsgType::FILTERADD)) { if (pfrom->nVersion >= NO_BLOOM_VERSION) { LOCK(cs_main); @@ -1935,7 +1935,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR } } - if (strCommand == NetMsgType::VERSION) { + if (msg_type == NetMsgType::VERSION) { // Each connection can only send one version message if (pfrom->nVersion != 0) { @@ -2107,7 +2107,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR // At this point, the outgoing message serialization version can't change. const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - if (strCommand == NetMsgType::VERACK) + if (msg_type == NetMsgType::VERACK) { pfrom->SetRecvVersion(std::min(pfrom->nVersion.load(), PROTOCOL_VERSION)); @@ -2152,7 +2152,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return false; } - if (strCommand == NetMsgType::ADDR) { + if (msg_type == NetMsgType::ADDR) { std::vector<CAddress> vAddr; vRecv >> vAddr; @@ -2192,7 +2192,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes - RelayAddress(addr, fReachable, connman); + RelayAddress(addr, fReachable, *connman); } // Do not store addresses outside our network if (fReachable) @@ -2206,13 +2206,13 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::SENDHEADERS) { + if (msg_type == NetMsgType::SENDHEADERS) { LOCK(cs_main); State(pfrom->GetId())->fPreferHeaders = true; return true; } - if (strCommand == NetMsgType::SENDCMPCT) { + if (msg_type == NetMsgType::SENDCMPCT) { bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 0; vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; @@ -2235,7 +2235,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::INV) { + if (msg_type == NetMsgType::INV) { std::vector<CInv> vInv; vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) @@ -2297,7 +2297,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETDATA) { + if (msg_type == NetMsgType::GETDATA) { std::vector<CInv> vInv; vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) @@ -2318,7 +2318,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETBLOCKS) { + if (msg_type == NetMsgType::GETBLOCKS) { CBlockLocator locator; uint256 hashStop; vRecv >> locator >> hashStop; @@ -2386,7 +2386,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETBLOCKTXN) { + if (msg_type == NetMsgType::GETBLOCKTXN) { BlockTransactionsRequest req; vRecv >> req; @@ -2435,7 +2435,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETHEADERS) { + if (msg_type == NetMsgType::GETHEADERS) { CBlockLocator locator; uint256 hashStop; vRecv >> locator >> hashStop; @@ -2502,7 +2502,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::TX) { + if (msg_type == NetMsgType::TX) { // Stop processing the transaction early if // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off // or if this peer is supposed to be a block-relay-only peer @@ -2644,7 +2644,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::CMPCTBLOCK) + if (msg_type == NetMsgType::CMPCTBLOCK) { // Ignore cmpctblock received while importing if (fImporting || fReindex) { @@ -2865,7 +2865,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::BLOCKTXN) + if (msg_type == NetMsgType::BLOCKTXN) { // Ignore blocktxn received while importing if (fImporting || fReindex) { @@ -2947,7 +2947,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::HEADERS) + if (msg_type == NetMsgType::HEADERS) { // Ignore headers received while importing if (fImporting || fReindex) { @@ -2973,7 +2973,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return ProcessHeadersMessage(pfrom, connman, mempool, headers, chainparams, /*via_compact_block=*/false); } - if (strCommand == NetMsgType::BLOCK) + if (msg_type == NetMsgType::BLOCK) { // Ignore block received while importing if (fImporting || fReindex) { @@ -3009,7 +3009,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETADDR) { + if (msg_type == NetMsgType::GETADDR) { // This asymmetric behavior for inbound and outbound connections was introduced // to prevent a fingerprinting attack: an attacker can send specific fake addresses // to users' AddrMan and later request them by sending getaddr messages. @@ -3043,7 +3043,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::MEMPOOL) { + if (msg_type == NetMsgType::MEMPOOL) { if (!(pfrom->GetLocalServices() & NODE_BLOOM) && !pfrom->HasPermission(PF_MEMPOOL)) { if (!pfrom->HasPermission(PF_NOBAN)) @@ -3071,7 +3071,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::PING) { + if (msg_type == NetMsgType::PING) { if (pfrom->nVersion > BIP0031_VERSION) { uint64_t nonce = 0; @@ -3092,7 +3092,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::PONG) { + if (msg_type == NetMsgType::PONG) { int64_t pingUsecEnd = nTimeReceived; uint64_t nonce = 0; size_t nAvail = vRecv.in_avail(); @@ -3148,7 +3148,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::FILTERLOAD) { + if (msg_type == NetMsgType::FILTERLOAD) { CBloomFilter filter; vRecv >> filter; @@ -3168,7 +3168,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::FILTERADD) { + if (msg_type == NetMsgType::FILTERADD) { std::vector<unsigned char> vData; vRecv >> vData; @@ -3192,19 +3192,19 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::FILTERCLEAR) { + if (msg_type == NetMsgType::FILTERCLEAR) { if (pfrom->m_tx_relay == nullptr) { return true; } LOCK(pfrom->m_tx_relay->cs_filter); if (pfrom->GetLocalServices() & NODE_BLOOM) { - pfrom->m_tx_relay->pfilter.reset(new CBloomFilter()); + pfrom->m_tx_relay->pfilter = nullptr; } pfrom->m_tx_relay->fRelayTxes = true; return true; } - if (strCommand == NetMsgType::FEEFILTER) { + if (msg_type == NetMsgType::FEEFILTER) { CAmount newFeeFilter = 0; vRecv >> newFeeFilter; if (MoneyRange(newFeeFilter)) { @@ -3217,7 +3217,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::NOTFOUND) { + if (msg_type == NetMsgType::NOTFOUND) { // Remove the NOTFOUND transactions from the peer LOCK(cs_main); CNodeState *state = State(pfrom->GetId()); @@ -3243,7 +3243,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR } // Ignore unknown commands for extensibility - LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->GetId()); + LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(msg_type), pfrom->GetId()); return true; } @@ -3338,7 +3338,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter LogPrint(BCLog::NET, "PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(msg.m_command), pfrom->GetId()); return fMoreWork; } - const std::string& strCommand = msg.m_command; + const std::string& msg_type = msg.m_command; // Message size unsigned int nMessageSize = msg.m_message_size; @@ -3348,7 +3348,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter if (!msg.m_valid_checksum) { LogPrint(BCLog::NET, "%s(%s, %u bytes): CHECKSUM ERROR peer=%d\n", __func__, - SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + SanitizeString(msg_type), nMessageSize, pfrom->GetId()); return fMoreWork; } @@ -3356,19 +3356,19 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fRet = false; try { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.m_time, chainparams, m_mempool, connman, m_banman, interruptMsgProc); + fRet = ProcessMessage(pfrom, msg_type, vRecv, msg.m_time, chainparams, m_mempool, connman, m_banman, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) fMoreWork = true; } catch (const std::exception& e) { - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what(), typeid(e).name()); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(msg_type), nMessageSize, e.what(), typeid(e).name()); } catch (...) { - LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(strCommand), nMessageSize); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(msg_type), nMessageSize); } if (!fRet) { - LogPrint(BCLog::NET, "%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + LogPrint(BCLog::NET, "%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(msg_type), nMessageSize, pfrom->GetId()); } LOCK(cs_main); @@ -3583,16 +3583,16 @@ bool PeerLogicValidation::SendMessages(CNode* pto) int64_t nNow = GetTimeMicros(); auto current_time = GetTime<std::chrono::microseconds>(); - if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { + if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->m_next_local_addr_send < current_time) { AdvertiseLocal(pto); - pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); + pto->m_next_local_addr_send = PoissonNextSend(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } // // Message: addr // - if (pto->IsAddrRelayPeer() && pto->nNextAddrSend < nNow) { - pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); + if (pto->IsAddrRelayPeer() && pto->m_next_addr_send < current_time) { + pto->m_next_addr_send = PoissonNextSend(current_time, AVG_ADDRESS_BROADCAST_INTERVAL); std::vector<CAddress> vAddr; vAddr.reserve(pto->vAddrToSend.size()); assert(pto->m_addr_known); diff --git a/src/net_processing.h b/src/net_processing.h index 65e3963c41..3d9bc65a3a 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -14,6 +14,7 @@ class CTxMemPool; extern RecursiveMutex cs_main; +extern RecursiveMutex g_cs_orphans; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 228caf74a9..6adb171abd 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/netaddress.h b/src/netaddress.h index b7381c1eb4..d0ab770379 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 641b2a5d9c..ec52a08ace 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/context.cpp b/src/node/context.cpp index 5b19a41bd4..0238aab0d9 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/context.h b/src/node/context.h index 1c592b456b..566ff170be 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <memory> #include <vector> +class ArgsManager; class BanMan; class CConnman; class CScheduler; @@ -33,6 +34,7 @@ struct NodeContext { CTxMemPool* mempool{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<PeerLogicValidation> peer_logic; std::unique_ptr<BanMan> banman; + ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<interfaces::Chain> chain; std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; std::unique_ptr<CScheduler> scheduler; diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index 5b16035f7d..c189018268 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/transaction.h b/src/node/transaction.h index a85dfb8ace..98d65bc4dd 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 71b5cba01c..ea7a86d6d6 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/prevector.h b/src/prevector.h index 6d690e7f96..9f2f7ba2de 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 6e72c1f15c..e6183cf2f4 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/protocol.cpp b/src/protocol.cpp index bd3ed25a8a..a3e844e35b 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/psbt.cpp b/src/psbt.cpp index e6b6285652..ef9781817a 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/psbt.h b/src/psbt.h index dfba261961..83a729217b 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 3ac98a5970..8110f4e895 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -55,7 +55,7 @@ struct AddressTableEntryLessThan static AddressTableEntry::Type translateTransactionType(const QString &strPurpose, bool isMine) { AddressTableEntry::Type addressType = AddressTableEntry::Hidden; - // "refund" addresses aren't shown, and change addresses aren't in mapAddressBook at all. + // "refund" addresses aren't shown, and change addresses aren't returned by getAddresses at all. if (strPurpose == "send") addressType = AddressTableEntry::Sending; else if (strPurpose == "receive") diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index 72f16bb09f..2739b21a9d 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index f01c506a1e..88d5b5811b 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 4313d6ee7f..8939b566f7 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -336,7 +336,7 @@ void BitcoinApplication::initializeResult(bool success) window->setClientModel(clientModel); #ifdef ENABLE_WALLET if (WalletModel::isWalletEnabled()) { - m_wallet_controller = new WalletController(m_node, platformStyle, optionsModel, this); + m_wallet_controller = new WalletController(*clientModel, platformStyle, this); window->setWalletController(m_wallet_controller); if (paymentServer) { paymentServer->setOptionsModel(optionsModel); @@ -381,7 +381,7 @@ void BitcoinApplication::shutdownResult() void BitcoinApplication::handleRunawayException(const QString &message) { - QMessageBox::critical(nullptr, "Runaway exception", BitcoinGUI::tr("A fatal error occurred. Bitcoin can no longer continue safely and will quit.") + QString("\n\n") + message); + QMessageBox::critical(nullptr, "Runaway exception", BitcoinGUI::tr("A fatal error occurred. %1 can no longer continue safely and will quit.").arg(PACKAGE_NAME) + QString("\n\n") + message); ::exit(EXIT_FAILURE); } diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 2918676c22..75f3e9bf45 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -456,7 +456,7 @@ void BitcoinGUI::createMenuBar() QAction* minimize_action = window_menu->addAction(tr("Minimize")); minimize_action->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_M)); connect(minimize_action, &QAction::triggered, [] { - qApp->focusWindow()->showMinimized(); + QApplication::activeWindow()->showMinimized(); }); connect(qApp, &QApplication::focusWindowChanged, [minimize_action] (QWindow* window) { minimize_action->setEnabled(window != nullptr && (window->flags() & Qt::Dialog) != Qt::Dialog && window->windowState() != Qt::WindowMinimized); diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 809cf8b4ed..9b5523f625 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index a1ec3eaab1..b94fcc9865 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -105,6 +105,14 @@ int64_t ClientModel::getHeaderTipTime() const return cachedBestHeaderTime; } +int ClientModel::getNumBlocks() const +{ + if (m_cached_num_blocks == -1) { + m_cached_num_blocks = m_node.getNumBlocks(); + } + return m_cached_num_blocks; +} + void ClientModel::updateNumConnections(int numConnections) { Q_EMIT numConnectionsChanged(numConnections); @@ -241,6 +249,8 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, int heig // cache best headers time and height to reduce future cs_main locks clientmodel->cachedBestHeaderHeight = height; clientmodel->cachedBestHeaderTime = blockTime; + } else { + clientmodel->m_cached_num_blocks = height; } // During initial sync, block notifications, and header notifications from reindexing are both throttled. diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 79175e0af4..7ac4120a8f 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -56,6 +56,7 @@ public: //! Return number of connections, default is in- and outbound (total) int getNumConnections(unsigned int flags = CONNECTIONS_ALL) const; + int getNumBlocks() const; int getHeaderTipHeight() const; int64_t getHeaderTipTime() const; @@ -73,9 +74,10 @@ public: bool getProxyInfo(std::string& ip_port) const; - // caches for the best header + // caches for the best header, number of blocks mutable std::atomic<int> cachedBestHeaderHeight; mutable std::atomic<int64_t> cachedBestHeaderTime; + mutable std::atomic<int> m_cached_num_blocks{-1}; private: interfaces::Node& m_node; diff --git a/src/qt/coincontroltreewidget.h b/src/qt/coincontroltreewidget.h index 39dc9a5e9e..86af555385 100644 --- a/src/qt/coincontroltreewidget.h +++ b/src/qt/coincontroltreewidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/csvmodelwriter.h b/src/qt/csvmodelwriter.h index e443529335..ad247b6859 100644 --- a/src/qt/csvmodelwriter.h +++ b/src/qt/csvmodelwriter.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 98dde1656a..a54b65961c 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/macnotificationhandler.mm b/src/qt/macnotificationhandler.mm index b16042e946..2d26a8383d 100644 --- a/src/qt/macnotificationhandler.mm +++ b/src/qt/macnotificationhandler.mm @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/main.cpp b/src/qt/main.cpp index 3dfd9e850e..6e772d58c5 100644 --- a/src/qt/main.cpp +++ b/src/qt/main.cpp @@ -1,10 +1,11 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/bitcoin.h> #include <util/translation.h> +#include <util/url.h> #include <QCoreApplication> @@ -15,5 +16,6 @@ extern const std::function<std::string(const char*)> G_TRANSLATION_FUN = [](const char* psz) { return QCoreApplication::translate("bitcoin-core", psz).toStdString(); }; +UrlDecodeFn* const URL_DECODE = urlDecode; int main(int argc, char* argv[]) { return GuiMain(argc, argv); } diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 6243a71c7d..49a1992803 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index a1fc791536..5220f8e138 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index cf45c5a08f..0e20b1d9dc 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/qvaluecombobox.h b/src/qt/qvaluecombobox.h index 5cca515079..bde9c0d1a6 100644 --- a/src/qt/qvaluecombobox.h +++ b/src/qt/qvaluecombobox.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/res/icons/bitcoin.ico b/src/qt/res/icons/bitcoin.ico Binary files differindex 8f5045015d..8f5045015d 100755..100644 --- a/src/qt/res/icons/bitcoin.ico +++ b/src/qt/res/icons/bitcoin.ico diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 0ffdc892c5..18a7e9b021 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index a8c82aaf6c..345c258845 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 4552753bf6..67c2c71e9c 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index e4ffa6cd9a..d8fe4a9238 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 0f082802cc..476128520c 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <interfaces/chain.h> #include <interfaces/node.h> +#include <qt/clientmodel.h> #include <qt/editaddressdialog.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> @@ -97,7 +98,7 @@ void TestAddAddressesToSendBook(interfaces::Node& node) auto check_addbook_size = [&wallet](int expected_size) { LOCK(wallet->cs_wallet); - QCOMPARE(static_cast<int>(wallet->mapAddressBook.size()), expected_size); + QCOMPARE(static_cast<int>(wallet->m_address_book.size()), expected_size); }; // We should start with the two addresses we added earlier and nothing else. @@ -106,8 +107,9 @@ void TestAddAddressesToSendBook(interfaces::Node& node) // Initialize relevant QT models. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); OptionsModel optionsModel(node); + ClientModel clientModel(node, &optionsModel); AddWallet(wallet); - WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); RemoveWallet(wallet); EditAddressDialog editAddressDialog(EditAddressDialog::NewSendingAddress); editAddressDialog.setModel(walletModel.getAddressTableModel()); diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 14a75b23f3..f88d57c716 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -82,6 +82,7 @@ void AppTests::appTests() // Reset global state to avoid interfering with later tests. AbortShutdown(); UnloadBlockIndex(); + WITH_LOCK(::cs_main, g_chainman.Reset()); } //! Entry point for BitcoinGUI tests. diff --git a/src/qt/test/apptests.h b/src/qt/test/apptests.h index d16c9fe487..f0a555005d 100644 --- a/src/qt/test/apptests.h +++ b/src/qt/test/apptests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/util.cpp b/src/qt/test/util.cpp index e09f0ad77d..759832381b 100644 --- a/src/qt/test/util.cpp +++ b/src/qt/test/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/util.h b/src/qt/test/util.h index 763847606a..950314da3b 100644 --- a/src/qt/test/util.h +++ b/src/qt/test/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index c1a0f63f73..94a1e0a63d 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <interfaces/chain.h> #include <interfaces/node.h> #include <qt/bitcoinamountfield.h> +#include <qt/clientmodel.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> #include <qt/qvalidatedlineedit.h> @@ -151,12 +152,9 @@ void TestGUI(interfaces::Node& node) wallet->SetLastBlockProcessed(105, ::ChainActive().Tip()->GetBlockHash()); } { - auto locked_chain = wallet->chain().lock(); - LockAssertion lock(::cs_main); - - WalletRescanReserver reserver(wallet.get()); + WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(locked_chain->getBlockHash(0), {} /* stop_block */, reserver, true /* fUpdate */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(Params().GetConsensus().hashGenesisBlock, 0 /* block height */, {} /* max height */, reserver, true /* fUpdate */); QCOMPARE(result.status, CWallet::ScanResult::SUCCESS); QCOMPARE(result.last_scanned_block, ::ChainActive().Tip()->GetBlockHash()); QVERIFY(result.last_failed_block.IsNull()); @@ -168,8 +166,9 @@ void TestGUI(interfaces::Node& node) SendCoinsDialog sendCoinsDialog(platformStyle.get()); TransactionView transactionView(platformStyle.get()); OptionsModel optionsModel(node); + ClientModel clientModel(node, &optionsModel); AddWallet(wallet); - WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); RemoveWallet(wallet); sendCoinsDialog.setModel(&walletModel); transactionView.setModel(&walletModel); diff --git a/src/qt/trafficgraphwidget.h b/src/qt/trafficgraphwidget.h index af5890ba24..7e8bfb2337 100644 --- a/src/qt/trafficgraphwidget.h +++ b/src/qt/trafficgraphwidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiondescdialog.h b/src/qt/transactiondescdialog.h index 74e34cde87..3204b22657 100644 --- a/src/qt/transactiondescdialog.h +++ b/src/qt/transactiondescdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 64e9c856db..18554aef1f 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -5,6 +5,7 @@ #include <qt/transactiontablemodel.h> #include <qt/addresstablemodel.h> +#include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> @@ -175,7 +176,7 @@ public: return cachedWallet.size(); } - TransactionRecord *index(interfaces::Wallet& wallet, int idx) + TransactionRecord *index(interfaces::Wallet& wallet, const int cur_num_blocks, const int idx) { if(idx >= 0 && idx < cachedWallet.size()) { @@ -191,7 +192,7 @@ public: interfaces::WalletTxStatus wtx; int numBlocks; int64_t block_time; - if (wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time) && rec->statusUpdateNeeded(numBlocks)) { + if (rec->statusUpdateNeeded(cur_num_blocks) && wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time)) { rec->updateStatus(wtx, numBlocks, block_time); } return rec; @@ -663,10 +664,10 @@ QVariant TransactionTableModel::headerData(int section, Qt::Orientation orientat QModelIndex TransactionTableModel::index(int row, int column, const QModelIndex &parent) const { Q_UNUSED(parent); - TransactionRecord *data = priv->index(walletModel->wallet(), row); + TransactionRecord *data = priv->index(walletModel->wallet(), walletModel->clientModel().getNumBlocks(), row); if(data) { - return createIndex(row, column, priv->index(walletModel->wallet(), row)); + return createIndex(row, column, data); } return QModelIndex(); } diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 3101fb01c3..d16feba1bb 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index 833b86fd3e..b6a42d3d9d 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index f076b5ba61..8e422fa962 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -1,10 +1,11 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/walletcontroller.h> #include <qt/askpassphrasedialog.h> +#include <qt/clientmodel.h> #include <qt/createwalletdialog.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> @@ -24,13 +25,14 @@ #include <QTimer> #include <QWindow> -WalletController::WalletController(interfaces::Node& node, const PlatformStyle* platform_style, OptionsModel* options_model, QObject* parent) +WalletController::WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent) : QObject(parent) , m_activity_thread(new QThread(this)) , m_activity_worker(new QObject) - , m_node(node) + , m_client_model(client_model) + , m_node(client_model.node()) , m_platform_style(platform_style) - , m_options_model(options_model) + , m_options_model(client_model.getOptionsModel()) { m_handler_load_wallet = m_node.handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { getOrCreateWallet(std::move(wallet)); @@ -104,7 +106,7 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal } // Instantiate model and register it. - WalletModel* wallet_model = new WalletModel(std::move(wallet), m_node, m_platform_style, m_options_model, nullptr); + WalletModel* wallet_model = new WalletModel(std::move(wallet), m_client_model, m_platform_style, nullptr); // Handler callback runs in a different thread so fix wallet model thread affinity. wallet_model->moveToThread(thread()); wallet_model->setParent(this); diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index f30eb25308..3808b7d8bf 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,6 +21,7 @@ #include <QTimer> #include <QString> +class ClientModel; class OptionsModel; class PlatformStyle; class WalletModel; @@ -47,7 +48,7 @@ class WalletController : public QObject void removeAndDeleteWallet(WalletModel* wallet_model); public: - WalletController(interfaces::Node& node, const PlatformStyle* platform_style, OptionsModel* options_model, QObject* parent); + WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent); ~WalletController(); //! Returns wallet models currently open. @@ -70,6 +71,7 @@ Q_SIGNALS: private: QThread* const m_activity_thread; QObject* const m_activity_worker; + ClientModel& m_client_model; interfaces::Node& m_node; const PlatformStyle* const m_platform_style; OptionsModel* const m_options_model; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 608797d6ad..e7581cd64e 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,6 +9,7 @@ #include <qt/walletmodel.h> #include <qt/addresstablemodel.h> +#include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> @@ -20,6 +21,7 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <key_io.h> +#include <psbt.h> #include <ui_interface.h> #include <util/system.h> // for GetBoolArg #include <wallet/coincontrol.h> @@ -33,8 +35,13 @@ #include <QTimer> -WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, interfaces::Node& node, const PlatformStyle *platformStyle, OptionsModel *_optionsModel, QObject *parent) : - QObject(parent), m_wallet(std::move(wallet)), m_node(node), optionsModel(_optionsModel), addressTableModel(nullptr), +WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent) : + QObject(parent), + m_wallet(std::move(wallet)), + m_client_model(client_model), + m_node(client_model.node()), + optionsModel(client_model.getOptionsModel()), + addressTableModel(nullptr), transactionTableModel(nullptr), recentRequestsTableModel(nullptr), cachedEncryptionStatus(Unencrypted), diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 7936014af9..07004b7c6b 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -24,6 +24,7 @@ enum class OutputType; class AddressTableModel; +class ClientModel; class OptionsModel; class PlatformStyle; class RecentRequestsTableModel; @@ -52,7 +53,7 @@ class WalletModel : public QObject Q_OBJECT public: - explicit WalletModel(std::unique_ptr<interfaces::Wallet> wallet, interfaces::Node& node, const PlatformStyle *platformStyle, OptionsModel *optionsModel, QObject *parent = nullptr); + explicit WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent = nullptr); ~WalletModel(); enum StatusCode // Returned by sendCoins @@ -143,6 +144,7 @@ public: interfaces::Node& node() const { return m_node; } interfaces::Wallet& wallet() const { return *m_wallet; } + ClientModel& clientModel() const { return m_client_model; } QString getWalletName() const; QString getDisplayName() const; @@ -159,6 +161,7 @@ private: std::unique_ptr<interfaces::Handler> m_handler_show_progress; std::unique_ptr<interfaces::Handler> m_handler_watch_only_changed; std::unique_ptr<interfaces::Handler> m_handler_can_get_addrs_changed; + ClientModel& m_client_model; interfaces::Node& m_node; bool fHaveWatchOnly; diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index bdcb82e06b..5b58e5fc1f 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/walletview.h b/src/qt/walletview.h index 78d870f59f..9100807c0a 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/random.cpp b/src/random.cpp index 2a27e6ba0d..765239e1f5 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/random.h b/src/random.h index 4e4597cff6..0c5008685b 100644 --- a/src/random.h +++ b/src/random.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 8b3d478529..073d82b491 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index c132f265d2..827b83d67e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -375,8 +375,6 @@ static UniValue getdifficulty(const JSONRPCRequest& request) static std::vector<RPCResult> MempoolEntryDescription() { return { RPCResult{RPCResult::Type::NUM, "vsize", "virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted."}, - RPCResult{RPCResult::Type::NUM, "size", "(DEPRECATED) same as vsize. Only returned if bitcoind is started with -deprecatedrpc=size\n" - "size will be completely removed in v0.20."}, RPCResult{RPCResult::Type::NUM, "weight", "transaction weight as defined in BIP 141."}, RPCResult{RPCResult::Type::STR_AMOUNT, "fee", "transaction fee in " + CURRENCY_UNIT + " (DEPRECATED)"}, RPCResult{RPCResult::Type::STR_AMOUNT, "modifiedfee", "transaction fee with fee deltas used for mining priority (DEPRECATED)"}, @@ -415,7 +413,6 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("fees", fees); info.pushKV("vsize", (int)e.GetTxSize()); - if (IsDeprecatedRPCEnabled("size")) info.pushKV("size", (int)e.GetTxSize()); info.pushKV("weight", (int)e.GetTxWeight()); info.pushKV("fee", ValueFromAmount(e.GetFee())); info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); @@ -812,7 +809,7 @@ static UniValue getblock(const JSONRPCRequest& request) "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, - {"verbosity", RPCArg::Type::NUM, /* default */ "1", "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, + {"verbosity|verbose", RPCArg::Type::NUM, /* default */ "1", "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, }, { RPCResult{"for verbosity = 0", @@ -1078,13 +1075,11 @@ UniValue gettxout(const JSONRPCRequest& request) static UniValue verifychain(const JSONRPCRequest& request) { - int nCheckLevel = gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL); - int nCheckDepth = gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS); RPCHelpMan{"verifychain", "\nVerifies blockchain database.\n", { - {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", nCheckLevel), "How thorough the block verification is."}, - {"nblocks", RPCArg::Type::NUM, /* default */ strprintf("%d, 0=all", nCheckDepth), "The number of blocks to check."}, + {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", DEFAULT_CHECKLEVEL), "How thorough the block verification is."}, + {"nblocks", RPCArg::Type::NUM, /* default */ strprintf("%d, 0=all", DEFAULT_CHECKBLOCKS), "The number of blocks to check."}, }, RPCResult{ RPCResult::Type::BOOL, "", "Verified or not"}, @@ -1094,15 +1089,12 @@ static UniValue verifychain(const JSONRPCRequest& request) }, }.Check(request); - LOCK(cs_main); + const int check_level(request.params[0].isNull() ? DEFAULT_CHECKLEVEL : request.params[0].get_int()); + const int check_depth{request.params[1].isNull() ? DEFAULT_CHECKBLOCKS : request.params[1].get_int()}; - if (!request.params[0].isNull()) - nCheckLevel = request.params[0].get_int(); - if (!request.params[1].isNull()) - nCheckDepth = request.params[1].get_int(); + LOCK(cs_main); - return CVerifyDB().VerifyDB( - Params(), &::ChainstateActive().CoinsTip(), nCheckLevel, nCheckDepth); + return CVerifyDB().VerifyDB(Params(), &::ChainstateActive().CoinsTip(), check_level, check_depth); } static void BuriedForkDescPushBack(UniValue& softforks, const std::string &name, int height) EXCLUSIVE_LOCKS_REQUIRED(cs_main) @@ -1319,7 +1311,7 @@ static UniValue getchaintips(const JSONRPCRequest& request) /* * Idea: the set of chain tips is ::ChainActive().tip, plus orphan blocks which do not have another orphan building off of them. * Algorithm: - * - Make one pass through g_blockman.m_block_index, picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. + * - Make one pass through BlockIndex(), picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. * - add ::ChainActive().Tip() */ @@ -1726,8 +1718,10 @@ static UniValue getblockstats(const JSONRPCRequest& request) {RPCResult::Type::NUM, "utxo_size_inc", "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, }}, RPCExamples{ - HelpExampleCli("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") - + HelpExampleRpc("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") + HelpExampleCli("getblockstats", R"('"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"' '["minfeerate","avgfeerate"]')") + + HelpExampleCli("getblockstats", R"(1000 '["minfeerate","avgfeerate"]')") + + HelpExampleRpc("getblockstats", R"("00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09", ["minfeerate","avgfeerate"])") + + HelpExampleRpc("getblockstats", R"(1000, ["minfeerate","avgfeerate"])") }, }.Check(request); @@ -2347,6 +2341,8 @@ UniValue dumptxoutset(const JSONRPCRequest& request) return result; } +void RegisterBlockchainRPCCommands(CRPCTable &t) +{ // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -2387,8 +2383,6 @@ static const CRPCCommand commands[] = }; // clang-format on -void RegisterBlockchainRPCCommands(CRPCTable &t) -{ for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) t.appendCommand(commands[vcidx].name, &commands[vcidx]); } diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index c1762483e9..5d0f2fd703 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -33,6 +33,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "generatetoaddress", 2, "maxtries" }, { "generatetodescriptor", 0, "num_blocks" }, { "generatetodescriptor", 2, "maxtries" }, + { "generateblock", 1, "transactions" }, { "getnetworkhashps", 0, "nblocks" }, { "getnetworkhashps", 1, "height" }, { "sendtoaddress", 1, "amount" }, @@ -97,10 +98,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "signrawtransactionwithkey", 1, "privkeys" }, { "signrawtransactionwithkey", 2, "prevtxs" }, { "signrawtransactionwithwallet", 1, "prevtxs" }, - { "sendrawtransaction", 1, "allowhighfees" }, { "sendrawtransaction", 1, "maxfeerate" }, { "testmempoolaccept", 0, "rawtxs" }, - { "testmempoolaccept", 1, "allowhighfees" }, { "testmempoolaccept", 1, "maxfeerate" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, @@ -154,6 +153,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "logging", 0, "include" }, { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, + { "upgradewallet", 0, "version" }, // Echo with conversion (For testing only) { "echojson", 0, "arg0" }, { "echojson", 1, "arg1" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index bde19d8e79..05d3fd6afb 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -101,6 +101,36 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].get_int() : 120, !request.params[1].isNull() ? request.params[1].get_int() : -1); } +static bool GenerateBlock(CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) +{ + block_hash.SetNull(); + + { + LOCK(cs_main); + IncrementExtraNonce(&block, ::ChainActive().Tip(), extra_nonce); + } + + CChainParams chainparams(Params()); + + while (max_tries > 0 && block.nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus()) && !ShutdownRequested()) { + ++block.nNonce; + --max_tries; + } + if (max_tries == 0 || ShutdownRequested()) { + return false; + } + if (block.nNonce == std::numeric_limits<uint32_t>::max()) { + return true; + } + + std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); + if (!ProcessNewBlock(chainparams, shared_pblock, true, nullptr)) + throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); + + block_hash = block.GetHash(); + return true; +} + static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { int nHeightEnd = 0; @@ -119,29 +149,54 @@ static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbas if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; - { - LOCK(cs_main); - IncrementExtraNonce(pblock, ::ChainActive().Tip(), nExtraNonce); - } - while (nMaxTries > 0 && pblock->nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(pblock->GetHash(), pblock->nBits, Params().GetConsensus()) && !ShutdownRequested()) { - ++pblock->nNonce; - --nMaxTries; - } - if (nMaxTries == 0 || ShutdownRequested()) { + + uint256 block_hash; + if (!GenerateBlock(*pblock, nMaxTries, nExtraNonce, block_hash)) { break; } - if (pblock->nNonce == std::numeric_limits<uint32_t>::max()) { - continue; + + if (!block_hash.IsNull()) { + ++nHeight; + blockHashes.push_back(block_hash.GetHex()); } - std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - if (!ProcessNewBlock(Params(), shared_pblock, true, nullptr)) - throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); - ++nHeight; - blockHashes.push_back(pblock->GetHash().GetHex()); } return blockHashes; } +static bool getScriptFromDescriptor(const std::string& descriptor, CScript& script, std::string& error) +{ + FlatSigningProvider key_provider; + const auto desc = Parse(descriptor, key_provider, error, /* require_checksum = */ false); + if (desc) { + if (desc->IsRange()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptor not accepted. Maybe pass through deriveaddresses first?"); + } + + FlatSigningProvider provider; + std::vector<CScript> scripts; + if (!desc->Expand(0, key_provider, scripts, provider)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); + } + + // Combo desriptors can have 2 or 4 scripts, so we can't just check scripts.size() == 1 + CHECK_NONFATAL(scripts.size() > 0 && scripts.size() <= 4); + + if (scripts.size() == 1) { + script = scripts.at(0); + } else if (scripts.size() == 4) { + // For uncompressed keys, take the 3rd script, since it is p2wpkh + script = scripts.at(2); + } else { + // Else take the 2nd script, since it is p2pkh + script = scripts.at(1); + } + + return true; + } else { + return false; + } +} + static UniValue generatetodescriptor(const JSONRPCRequest& request) { RPCHelpMan{ @@ -166,27 +221,15 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) const int num_blocks{request.params[0].get_int()}; const int64_t max_tries{request.params[2].isNull() ? 1000000 : request.params[2].get_int()}; - FlatSigningProvider key_provider; + CScript coinbase_script; std::string error; - const auto desc = Parse(request.params[1].get_str(), key_provider, error, /* require_checksum = */ false); - if (!desc) { + if (!getScriptFromDescriptor(request.params[1].get_str(), coinbase_script, error)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); } - if (desc->IsRange()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptor not accepted. Maybe pass through deriveaddresses first?"); - } - - FlatSigningProvider provider; - std::vector<CScript> coinbase_script; - if (!desc->Expand(0, key_provider, coinbase_script, provider)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); - } const CTxMemPool& mempool = EnsureMemPool(); - CHECK_NONFATAL(coinbase_script.size() == 1); - - return generateBlocks(mempool, coinbase_script.at(0), num_blocks, max_tries); + return generateBlocks(mempool, coinbase_script, num_blocks, max_tries); } static UniValue generatetoaddress(const JSONRPCRequest& request) @@ -229,6 +272,113 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) return generateBlocks(mempool, coinbase_script, nGenerate, nMaxTries); } +static UniValue generateblock(const JSONRPCRequest& request) +{ + RPCHelpMan{"generateblock", + "\nMine a block with a set of ordered transactions immediately to a specified address or descriptor (before the RPC call returns)\n", + { + {"output", RPCArg::Type::STR, RPCArg::Optional::NO, "The address or descriptor to send the newly generated bitcoin to."}, + {"transactions", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings which are either txids or raw transactions.\n" + "Txids must reference transactions currently in the mempool.\n" + "All transactions must be valid and in valid order, otherwise the block will be rejected.", + { + {"rawtx/txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, + }, + }, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "hash", "hash of generated block"}, + } + }, + RPCExamples{ + "\nGenerate a block to myaddress, with txs rawtx and mempool_txid\n" + + HelpExampleCli("generateblock", R"("myaddress" '["rawtx", "mempool_txid"]')") + }, + }.Check(request); + + const auto address_or_descriptor = request.params[0].get_str(); + CScript coinbase_script; + std::string error; + + if (!getScriptFromDescriptor(address_or_descriptor, coinbase_script, error)) { + const auto destination = DecodeDestination(address_or_descriptor); + if (!IsValidDestination(destination)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address or descriptor"); + } + + coinbase_script = GetScriptForDestination(destination); + } + + const CTxMemPool& mempool = EnsureMemPool(); + + std::vector<CTransactionRef> txs; + const auto raw_txs_or_txids = request.params[1].get_array(); + for (size_t i = 0; i < raw_txs_or_txids.size(); i++) { + const auto str(raw_txs_or_txids[i].get_str()); + + uint256 hash; + CMutableTransaction mtx; + if (ParseHashStr(str, hash)) { + + const auto tx = mempool.get(hash); + if (!tx) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Transaction %s not in mempool.", str)); + } + + txs.emplace_back(tx); + + } else if (DecodeHexTx(mtx, str)) { + txs.push_back(MakeTransactionRef(std::move(mtx))); + + } else { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("Transaction decode failed for %s", str)); + } + } + + CChainParams chainparams(Params()); + CBlock block; + + { + LOCK(cs_main); + + CTxMemPool empty_mempool; + std::unique_ptr<CBlockTemplate> blocktemplate(BlockAssembler(empty_mempool, chainparams).CreateNewBlock(coinbase_script)); + if (!blocktemplate) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); + } + block = blocktemplate->block; + } + + CHECK_NONFATAL(block.vtx.size() == 1); + + // Add transactions + block.vtx.insert(block.vtx.end(), txs.begin(), txs.end()); + RegenerateCommitments(block); + + { + LOCK(cs_main); + + BlockValidationState state; + if (!TestBlockValidity(state, chainparams, block, LookupBlockIndex(block.hashPrevBlock), false, false)) { + throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("TestBlockValidity failed: %s", state.ToString())); + } + } + + uint256 block_hash; + uint64_t max_tries{1000000}; + unsigned int extra_nonce{0}; + + if (!GenerateBlock(block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { + throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block."); + } + + UniValue obj(UniValue::VOBJ); + obj.pushKV("hash", block_hash.GetHex()); + return obj; +} + static UniValue getmininginfo(const JSONRPCRequest& request) { RPCHelpMan{"getmininginfo", @@ -1022,6 +1172,8 @@ static UniValue estimaterawfee(const JSONRPCRequest& request) return result; } +void RegisterMiningRPCCommands(CRPCTable &t) +{ // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -1036,6 +1188,7 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, {"nblocks","address","maxtries"} }, { "generating", "generatetodescriptor", &generatetodescriptor, {"num_blocks","descriptor","maxtries"} }, + { "generating", "generateblock", &generateblock, {"output","transactions"} }, { "util", "estimatesmartfee", &estimatesmartfee, {"conf_target", "estimate_mode"} }, @@ -1043,8 +1196,6 @@ static const CRPCCommand commands[] = }; // clang-format on -void RegisterMiningRPCCommands(CRPCTable &t) -{ for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) t.appendCommand(commands[vcidx].name, &commands[vcidx]); } diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index c87c1a5418..0d51c6ae1e 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -1,9 +1,10 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <httpserver.h> +#include <interfaces/chain.h> #include <key_io.h> #include <node/context.h> #include <outputtype.h> @@ -363,7 +364,13 @@ static UniValue setmocktime(const JSONRPCRequest& request) LOCK(cs_main); RPCTypeCheck(request.params, {UniValue::VNUM}); - SetMockTime(request.params[0].get_int64()); + int64_t time = request.params[0].get_int64(); + SetMockTime(time); + if (g_rpc_node) { + for (const auto& chain_client : g_rpc_node->chain_clients) { + chain_client->setMockTime(time); + } + } return NullUniValue; } @@ -589,6 +596,8 @@ static UniValue echo(const JSONRPCRequest& request) return request.params; } +void RegisterMiscRPCCommands(CRPCTable &t) +{ // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -610,8 +619,6 @@ static const CRPCCommand commands[] = }; // clang-format on -void RegisterMiscRPCCommands(CRPCTable &t) -{ for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) t.appendCommand(commands[vcidx].name, &commands[vcidx]); } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index caa62ca958..d6d15f8b56 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -760,6 +760,8 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) return ret; } +void RegisterNetRPCCommands(CRPCTable &t) +{ // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -780,8 +782,6 @@ static const CRPCCommand commands[] = }; // clang-format on -void RegisterNetRPCCommands(CRPCTable &t) -{ for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) t.appendCommand(commands[vcidx].name, &commands[vcidx]); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index ae3f15cec2..2f4e2916e2 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -825,7 +825,7 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) RPCTypeCheck(request.params, { UniValue::VSTR, - UniValueType(), // NUM or BOOL, checked later + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() }); // parse hex string from parameter @@ -834,13 +834,9 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - CFeeRate max_raw_tx_fee_rate = DEFAULT_MAX_RAW_TX_FEE_RATE; - // TODO: temporary migration code for old clients. Remove in v0.20 - if (request.params[1].isBool()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); - } else if (!request.params[1].isNull()) { - max_raw_tx_fee_rate = CFeeRate(AmountFromValue(request.params[1])); - } + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); int64_t virtual_size = GetVirtualTransactionSize(*tx); CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); @@ -896,7 +892,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) RPCTypeCheck(request.params, { UniValue::VARR, - UniValueType(), // NUM or BOOL, checked later + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() }); if (request.params[0].get_array().size() != 1) { @@ -910,13 +906,9 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) CTransactionRef tx(MakeTransactionRef(std::move(mtx))); const uint256& tx_hash = tx->GetHash(); - CFeeRate max_raw_tx_fee_rate = DEFAULT_MAX_RAW_TX_FEE_RATE; - // TODO: temporary migration code for old clients. Remove in v0.20 - if (request.params[1].isBool()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); - } else if (!request.params[1].isNull()) { - max_raw_tx_fee_rate = CFeeRate(AmountFromValue(request.params[1])); - } + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); CTxMemPool& mempool = EnsureMemPool(); int64_t virtual_size = GetVirtualTransactionSize(*tx); @@ -1813,6 +1805,8 @@ UniValue analyzepsbt(const JSONRPCRequest& request) return result; } +void RegisterRawTransactionRPCCommands(CRPCTable &t) +{ // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -1821,10 +1815,10 @@ static const CRPCCommand commands[] = { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, - { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","allowhighfees|maxfeerate"} }, + { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","maxfeerate"} }, { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, { "rawtransactions", "signrawtransactionwithkey", &signrawtransactionwithkey, {"hexstring","privkeys","prevtxs","sighashtype"} }, - { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","allowhighfees|maxfeerate"} }, + { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","maxfeerate"} }, { "rawtransactions", "decodepsbt", &decodepsbt, {"psbt"} }, { "rawtransactions", "combinepsbt", &combinepsbt, {"txs"} }, { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, @@ -1839,8 +1833,6 @@ static const CRPCCommand commands[] = }; // clang-format on -void RegisterRawTransactionRPCCommands(CRPCTable &t) -{ for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) t.appendCommand(commands[vcidx].name, &commands[vcidx]); } diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index 54baec6c6f..bd82773bf2 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -216,7 +216,7 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst keystore->AddCScript(script); // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). // This is done for redeemScript only for compatibility, it is encouraged to use the explicit witnessScript field instead. - CScript witness_output_script{GetScriptForWitness(script)}; + CScript witness_output_script{GetScriptForDestination(WitnessV0ScriptHash(script))}; keystore->AddCScript(witness_output_script); if (!ws.isNull() && !rs.isNull()) { diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 436db5dc60..942314eccf 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 7e1fb7a59d..860fa198d5 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,6 +13,9 @@ #include <tuple> +#include <boost/algorithm/string/classification.hpp> +#include <boost/algorithm/string/split.hpp> + const std::string UNIX_EPOCH_TIME = "UNIX epoch time"; const std::string EXAMPLE_ADDRESS[2] = {"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl", "bc1q02ad21edsxd23d32dfgqqsz4vv4nmtfzuklhy3"}; @@ -330,7 +333,7 @@ struct Sections { if (outer_type == OuterType::NONE) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; if (arg.m_type_str.size() != 0 && push_name) { - left += "\"" + arg.m_name + "\": " + arg.m_type_str.at(0); + left += "\"" + arg.GetName() + "\": " + arg.m_type_str.at(0); } else { left += push_name ? arg.ToStringObj(/* oneline */ false) : arg.ToString(/* oneline */ false); } @@ -341,7 +344,7 @@ struct Sections { case RPCArg::Type::OBJ: case RPCArg::Type::OBJ_USER_KEYS: { const auto right = outer_type == OuterType::NONE ? "" : arg.ToDescriptionString(); - PushSection({indent + (push_name ? "\"" + arg.m_name + "\": " : "") + "{", right}); + PushSection({indent + (push_name ? "\"" + arg.GetName() + "\": " : "") + "{", right}); for (const auto& arg_inner : arg.m_inner) { Push(arg_inner, current_indent + 2, OuterType::OBJ); } @@ -353,7 +356,7 @@ struct Sections { } case RPCArg::Type::ARR: { auto left = indent; - left += push_name ? "\"" + arg.m_name + "\": " : ""; + left += push_name ? "\"" + arg.GetName() + "\": " : ""; left += "["; const auto right = outer_type == OuterType::NONE ? "" : arg.ToDescriptionString(); PushSection({left, right}); @@ -419,8 +422,12 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP { std::set<std::string> named_args; for (const auto& arg : m_args) { + std::vector<std::string> names; + boost::split(names, arg.m_names, boost::is_any_of("|")); // Should have unique named arguments - CHECK_NONFATAL(named_args.insert(arg.m_name).second); + for (const std::string& name : names) { + CHECK_NONFATAL(named_args.insert(name).second); + } } } @@ -489,7 +496,7 @@ std::string RPCHelpMan::ToString() const if (i == 0) ret += "\nArguments:\n"; // Push named argument name and description - sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.m_name, arg.ToDescriptionString()); + sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString()); sections.m_max_pad = std::max(sections.m_max_pad, sections.m_sections.back().m_left.size()); // Recursively push nested args @@ -506,6 +513,17 @@ std::string RPCHelpMan::ToString() const return ret; } +std::string RPCArg::GetFirstName() const +{ + return m_names.substr(0, m_names.find("|")); +} + +std::string RPCArg::GetName() const +{ + CHECK_NONFATAL(std::string::npos == m_names.find("|")); + return m_names; +} + bool RPCArg::IsOptional() const { if (m_fallback.which() == 1) { @@ -681,7 +699,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const { std::string res; res += "\""; - res += m_name; + res += GetFirstName(); if (oneline) { res += "\":"; } else { @@ -723,13 +741,13 @@ std::string RPCArg::ToString(const bool oneline) const switch (m_type) { case Type::STR_HEX: case Type::STR: { - return "\"" + m_name + "\""; + return "\"" + GetFirstName() + "\""; } case Type::NUM: case Type::RANGE: case Type::AMOUNT: case Type::BOOL: { - return m_name; + return GetFirstName(); } case Type::OBJ: case Type::OBJ_USER_KEYS: { diff --git a/src/rpc/util.h b/src/rpc/util.h index f65ad1246b..53dce2c397 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -142,7 +142,7 @@ struct RPCArg { OMITTED, }; using Fallback = boost::variant<Optional, /* default value for optional args */ std::string>; - const std::string m_name; //!< The name of the arg (can be empty for inner args) + const std::string m_names; //!< The name of the arg (can be empty for inner args, can contain multiple aliases separated by | for named request arguments) const Type m_type; const std::vector<RPCArg> m_inner; //!< Only used for arrays or dicts const Fallback m_fallback; @@ -157,7 +157,7 @@ struct RPCArg { const std::string description, const std::string oneline_description = "", const std::vector<std::string> type_str = {}) - : m_name{std::move(name)}, + : m_names{std::move(name)}, m_type{std::move(type)}, m_fallback{std::move(fallback)}, m_description{std::move(description)}, @@ -175,7 +175,7 @@ struct RPCArg { const std::vector<RPCArg> inner, const std::string oneline_description = "", const std::vector<std::string> type_str = {}) - : m_name{std::move(name)}, + : m_names{std::move(name)}, m_type{std::move(type)}, m_inner{std::move(inner)}, m_fallback{std::move(fallback)}, @@ -188,6 +188,12 @@ struct RPCArg { bool IsOptional() const; + /** Return the first of all aliases */ + std::string GetFirstName() const; + + /** Return the name, throws when there are aliases */ + std::string GetName() const; + /** * Return the type string of the argument. * Set oneline to allow it to be overridden by a custom oneline type string (m_oneline_description). diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 4cac5a54e0..c4bd47310b 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 83dc046ca1..68d83299ee 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 34cd5760de..69850eb670 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 083022fbdd..23d5b72a5c 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -342,6 +342,35 @@ public: }; } +/** Helper for OP_CHECKSIG and OP_CHECKSIGVERIFY + * + * A return value of false means the script fails entirely. When true is returned, the + * fSuccess variable indicates whether the signature check itself succeeded. + */ +static bool EvalChecksig(const valtype& vchSig, const valtype& vchPubKey, CScript::const_iterator pbegincodehash, CScript::const_iterator pend, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* serror, bool& fSuccess) +{ + // Subset of script starting at the most recent codeseparator + CScript scriptCode(pbegincodehash, pend); + + // Drop the signature in pre-segwit scripts but not segwit scripts + if (sigversion == SigVersion::BASE) { + int found = FindAndDelete(scriptCode, CScript() << vchSig); + if (found > 0 && (flags & SCRIPT_VERIFY_CONST_SCRIPTCODE)) + return set_error(serror, SCRIPT_ERR_SIG_FINDANDDELETE); + } + + if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, sigversion, serror)) { + //serror is set + return false; + } + fSuccess = checker.CheckSig(vchSig, vchPubKey, scriptCode, sigversion); + + if (!fSuccess && (flags & SCRIPT_VERIFY_NULLFAIL) && vchSig.size()) + return set_error(serror, SCRIPT_ERR_SIG_NULLFAIL); + + return true; +} + bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& script, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* serror) { static const CScriptNum bnZero(0); @@ -985,25 +1014,8 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& valtype& vchSig = stacktop(-2); valtype& vchPubKey = stacktop(-1); - // Subset of script starting at the most recent codeseparator - CScript scriptCode(pbegincodehash, pend); - - // Drop the signature in pre-segwit scripts but not segwit scripts - if (sigversion == SigVersion::BASE) { - int found = FindAndDelete(scriptCode, CScript() << vchSig); - if (found > 0 && (flags & SCRIPT_VERIFY_CONST_SCRIPTCODE)) - return set_error(serror, SCRIPT_ERR_SIG_FINDANDDELETE); - } - - if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, sigversion, serror)) { - //serror is set - return false; - } - bool fSuccess = checker.CheckSig(vchSig, vchPubKey, scriptCode, sigversion); - - if (!fSuccess && (flags & SCRIPT_VERIFY_NULLFAIL) && vchSig.size()) - return set_error(serror, SCRIPT_ERR_SIG_NULLFAIL); - + bool fSuccess = true; + if (!EvalChecksig(vchSig, vchPubKey, pbegincodehash, pend, flags, checker, sigversion, serror, fSuccess)) return false; popstack(stack); popstack(stack); stack.push_back(fSuccess ? vchTrue : vchFalse); @@ -1279,18 +1291,29 @@ uint256 GetOutputsHash(const T& txTo) } // namespace template <class T> -PrecomputedTransactionData::PrecomputedTransactionData(const T& txTo) +void PrecomputedTransactionData::Init(const T& txTo) { + assert(!m_ready); + // Cache is calculated only for transactions with witness if (txTo.HasWitness()) { hashPrevouts = GetPrevoutHash(txTo); hashSequence = GetSequenceHash(txTo); hashOutputs = GetOutputsHash(txTo); - ready = true; } + + m_ready = true; +} + +template <class T> +PrecomputedTransactionData::PrecomputedTransactionData(const T& txTo) +{ + Init(txTo); } // explicit instantiation +template void PrecomputedTransactionData::Init(const CTransaction& txTo); +template void PrecomputedTransactionData::Init(const CMutableTransaction& txTo); template PrecomputedTransactionData::PrecomputedTransactionData(const CTransaction& txTo); template PrecomputedTransactionData::PrecomputedTransactionData(const CMutableTransaction& txTo); @@ -1303,7 +1326,7 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn uint256 hashPrevouts; uint256 hashSequence; uint256 hashOutputs; - const bool cacheready = cache && cache->ready; + const bool cacheready = cache && cache->m_ready; if (!(nHashType & SIGHASH_ANYONECANPAY)) { hashPrevouts = cacheready ? cache->hashPrevouts : GetPrevoutHash(txTo); diff --git a/src/script/interpreter.h b/src/script/interpreter.h index 2b104a608c..71f2436369 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -121,7 +121,12 @@ bool CheckSignatureEncoding(const std::vector<unsigned char> &vchSig, unsigned i struct PrecomputedTransactionData { uint256 hashPrevouts, hashSequence, hashOutputs; - bool ready = false; + bool m_ready = false; + + PrecomputedTransactionData() = default; + + template <class T> + void Init(const T& tx); template <class T> explicit PrecomputedTransactionData(const T& tx); diff --git a/src/script/script.h b/src/script/script.h index 7aaa10b60b..58f0818925 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -433,7 +433,7 @@ public: return ret; } - CScript(int64_t b) { operator<<(b); } + explicit CScript(int64_t b) { operator<<(b); } explicit CScript(opcodetype b) { operator<<(b); } explicit CScript(const CScriptNum& b) { operator<<(b); } diff --git a/src/script/script_error.cpp b/src/script/script_error.cpp index ff521d5860..57e8fee539 100644 --- a/src/script/script_error.cpp +++ b/src/script/script_error.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sign.cpp b/src/script/sign.cpp index fe8292fe57..1e00afcf89 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sign.h b/src/script/sign.h index f03af0713f..b77d26c0d7 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/serialize.h b/src/serialize.h index 5045cb3c7f..fe53eeed31 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/span.h b/src/span.h index 9379b15c81..664ea8d4c6 100644 --- a/src/span.h +++ b/src/span.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index f3cc12201c..9b14637b95 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/sync.h b/src/sync.h index ead2cdc67b..0c6f0ef0a7 100644 --- a/src/sync.h +++ b/src/sync.h @@ -210,7 +210,7 @@ public: friend class reverse_lock; }; -#define REVERSE_LOCK(g) decltype(g)::reverse_lock PASTE2(revlock, __COUNTER__)(g, #g, __FILE__, __LINE__) +#define REVERSE_LOCK(g) typename std::decay<decltype(g)>::type::reverse_lock PASTE2(revlock, __COUNTER__)(g, #g, __FILE__, __LINE__) template<typename MutexArg> using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove_pointer<MutexArg>::type>::type>; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index dfa8a6df21..bc6b38c682 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -1,18 +1,19 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> +#include <test/data/asmap.raw.h> #include <test/util/setup_common.h> -#include <string> -#include <boost/test/unit_test.hpp> #include <util/asmap.h> #include <util/string.h> -#include <test/data/asmap.raw.h> - #include <hash.h> #include <netbase.h> #include <random.h> +#include <boost/test/unit_test.hpp> + +#include <string> + class CAddrManTest : public CAddrMan { private: diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index 3723a48903..a135c93786 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -1,17 +1,19 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <arith_uint256.h> +#include <test/util/setup_common.h> +#include <uint256.h> + #include <boost/test/unit_test.hpp> -#include <stdint.h> -#include <sstream> + +#include <cmath> #include <iomanip> #include <limits> -#include <cmath> -#include <uint256.h> -#include <arith_uint256.h> +#include <sstream> +#include <stdint.h> #include <string> -#include <test/util/setup_common.h> BOOST_FIXTURE_TEST_SUITE(arith_uint256_tests, BasicTestingSetup) diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index 690368b177..eedab30576 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 94df4d1955..5927eab6cf 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index 53df032252..32329eb510 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2019 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <key.h> #include <key_io.h> #include <streams.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <string> #include <vector> diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index aa704642bf..c8e8cdeeb3 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <chain.h> #include <rpc/blockchain.h> -#include <util/string.h> #include <test/util/setup_common.h> +#include <util/string.h> /* Equality between doubles is imprecise. Comparison should be done * with a small threshold of tolerance, rather than exact equality. diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 5e52dc268f..ff01f730a3 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 4a7ad9b38b..a75d9fc9ec 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,10 +12,10 @@ #include <random.h> #include <serialize.h> #include <streams.h> +#include <test/util/setup_common.h> #include <uint256.h> -#include <util/system.h> #include <util/strencodings.h> -#include <test/util/setup_common.h> +#include <util/system.h> #include <vector> @@ -27,6 +27,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) { CBloomFilter filter(3, 0.01, 0, BLOOM_UPDATE_ALL); + BOOST_CHECK_MESSAGE( !filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter should be empty!"); filter.insert(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")); BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter doesn't contain just-inserted object!"); // One bit different in first byte @@ -50,8 +51,6 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter doesn't contain just-inserted object!"); - filter.clear(); - BOOST_CHECK_MESSAGE( !filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter should be empty!"); } BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) diff --git a/src/test/bswap_tests.cpp b/src/test/bswap_tests.cpp index 0b4bfdb019..c89cb5488d 100644 --- a/src/test/bswap_tests.cpp +++ b/src/test/bswap_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index a9628e85f9..0565982215 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -1,23 +1,23 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <checkqueue.h> +#include <test/util/setup_common.h> #include <util/memory.h> #include <util/system.h> #include <util/time.h> -#include <test/util/setup_common.h> -#include <checkqueue.h> #include <boost/test/unit_test.hpp> #include <boost/thread.hpp> + #include <atomic> -#include <thread> -#include <vector> -#include <mutex> #include <condition_variable> - +#include <mutex> +#include <thread> #include <unordered_set> #include <utility> +#include <vector> BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, TestingSetup) diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 436c1bffa0..c91621e227 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -8,6 +8,7 @@ #include <script/standard.h> #include <streams.h> #include <test/util/setup_common.h> +#include <txdb.h> #include <uint256.h> #include <undo.h> #include <util/strencodings.h> @@ -109,7 +110,12 @@ static const unsigned int NUM_SIMULATION_ITERATIONS = 40000; // // During the process, booleans are kept to make sure that the randomized // operation hits all branches. -BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) +// +// If fake_best_block is true, assign a random uint256 to mock the recording +// of best block on flush. This is necessary when using CCoinsViewDB as the base, +// otherwise we'll hit an assertion in BatchWrite. +// +void SimulationTest(CCoinsView* base, bool fake_best_block) { // Various coverage trackers. bool removed_all_caches = false; @@ -126,9 +132,8 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) std::map<COutPoint, Coin> result; // The cache stack. - CCoinsViewTest base; // A CCoinsViewTest at the bottom. std::vector<CCoinsViewCacheTest*> stack; // A stack of CCoinsViewCaches on top. - stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. + stack.push_back(new CCoinsViewCacheTest(base)); // Start with one cache. // Use a limited set of random transaction ids, so we do test overwriting entries. std::vector<uint256> txids; @@ -211,6 +216,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Every 100 iterations, flush an intermediate cache if (stack.size() > 1 && InsecureRandBool() == 0) { unsigned int flushIndex = InsecureRandRange(stack.size() - 1); + if (fake_best_block) stack[flushIndex]->SetBestBlock(InsecureRand256()); BOOST_CHECK(stack[flushIndex]->Flush()); } } @@ -218,13 +224,14 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Every 100 iterations, change the cache stack. if (stack.size() > 0 && InsecureRandBool() == 0) { //Remove the top cache + if (fake_best_block) stack.back()->SetBestBlock(InsecureRand256()); BOOST_CHECK(stack.back()->Flush()); delete stack.back(); stack.pop_back(); } if (stack.size() == 0 || (stack.size() < 4 && InsecureRandBool())) { //Add a new cache - CCoinsView* tip = &base; + CCoinsView* tip = base; if (stack.size() > 0) { tip = stack.back(); } else { @@ -256,6 +263,16 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) BOOST_CHECK(uncached_an_entry); } +// Run the above simulation for multiple base types. +BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) +{ + CCoinsViewTest base; + SimulationTest(&base, false); + + CCoinsViewDB db_base{"test", /*nCacheSize*/ 1 << 23, /*fMemory*/ true, /*fWipe*/ false}; + SimulationTest(&db_base, true); +} + // Store of all necessary tx and undo data for next test typedef std::map<COutPoint, std::tuple<CTransaction,CTxUndo,Coin>> UtxoData; UtxoData utxoData; diff --git a/src/test/compilerbug_tests.cpp b/src/test/compilerbug_tests.cpp index 1a6fcda009..b68bc279e1 100644 --- a/src/test/compilerbug_tests.cpp +++ b/src/test/compilerbug_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> +#include <test/util/setup_common.h> BOOST_FIXTURE_TEST_SUITE(compilerbug_tests, BasicTestingSetup) diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 22eae91cf0..df1a119d79 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compressor.h> -#include <test/util/setup_common.h> #include <script/standard.h> +#include <test/util/setup_common.h> #include <stdint.h> diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 2deb0c5bfc..f64251fe32 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <crypto/aes.h> #include <crypto/chacha20.h> #include <crypto/chacha_poly_aead.h> -#include <crypto/poly1305.h> #include <crypto/hkdf_sha256_32.h> #include <crypto/hmac_sha256.h> #include <crypto/hmac_sha512.h> +#include <crypto/poly1305.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> #include <crypto/sha512.h> #include <random.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <vector> diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 6be24c0845..3a951d28ae 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> #include <cuckoocache.h> +#include <deque> +#include <random.h> #include <script/sigcache.h> #include <test/util/setup_common.h> -#include <random.h> #include <thread> -#include <deque> /** Test Suite for CuckooCache * diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 3dfae29de6..c378546e8b 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <dbwrapper.h> -#include <uint256.h> #include <test/util/setup_common.h> +#include <uint256.h> #include <util/memory.h> #include <memory> diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 7310498eb6..6314c1a42f 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -52,7 +52,6 @@ struct COrphanTx { NodeId fromPeer; int64_t nTimeExpire; }; -extern RecursiveMutex g_cs_orphans; extern std::map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(g_cs_orphans); static CService ip(uint32_t i) diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 3154c619d2..121b80ab2d 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -1,16 +1,18 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <vector> -#include <string> +#include <script/descriptor.h> #include <script/sign.h> #include <script/standard.h> #include <test/util/setup_common.h> -#include <boost/test/unit_test.hpp> -#include <script/descriptor.h> #include <util/strencodings.h> +#include <boost/test/unit_test.hpp> + +#include <string> +#include <vector> + namespace { void CheckUnparsable(const std::string& prv, const std::string& pub, const std::string& expected_error) diff --git a/src/test/fuzz/addition_overflow.cpp b/src/test/fuzz/addition_overflow.cpp new file mode 100644 index 0000000000..a455992b13 --- /dev/null +++ b/src/test/fuzz/addition_overflow.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +#if defined(__has_builtin) +#if __has_builtin(__builtin_add_overflow) +#define HAVE_BUILTIN_ADD_OVERFLOW +#endif +#elif defined(__GNUC__) && (__GNUC__ >= 5) +#define HAVE_BUILTIN_ADD_OVERFLOW +#endif + +namespace { +template <typename T> +void TestAdditionOverflow(FuzzedDataProvider& fuzzed_data_provider) +{ + const T i = fuzzed_data_provider.ConsumeIntegral<T>(); + const T j = fuzzed_data_provider.ConsumeIntegral<T>(); + const bool is_addition_overflow_custom = AdditionOverflow(i, j); +#if defined(HAVE_BUILTIN_ADD_OVERFLOW) + T result_builtin; + const bool is_addition_overflow_builtin = __builtin_add_overflow(i, j, &result_builtin); + assert(is_addition_overflow_custom == is_addition_overflow_builtin); + if (!is_addition_overflow_custom) { + assert(i + j == result_builtin); + } +#else + if (!is_addition_overflow_custom) { + (void)(i + j); + } +#endif +} +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + TestAdditionOverflow<int64_t>(fuzzed_data_provider); + TestAdditionOverflow<uint64_t>(fuzzed_data_provider); + TestAdditionOverflow<int32_t>(fuzzed_data_provider); + TestAdditionOverflow<uint32_t>(fuzzed_data_provider); + TestAdditionOverflow<int16_t>(fuzzed_data_provider); + TestAdditionOverflow<uint16_t>(fuzzed_data_provider); + TestAdditionOverflow<char>(fuzzed_data_provider); + TestAdditionOverflow<unsigned char>(fuzzed_data_provider); + TestAdditionOverflow<signed char>(fuzzed_data_provider); +} diff --git a/src/test/fuzz/base_encode_decode.cpp b/src/test/fuzz/base_encode_decode.cpp index adad6b3f96..8d49f93c2f 100644 --- a/src/test/fuzz/base_encode_decode.cpp +++ b/src/test/fuzz/base_encode_decode.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,8 +6,8 @@ #include <base58.h> #include <psbt.h> -#include <util/string.h> #include <util/strencodings.h> +#include <util/string.h> #include <cassert> #include <cstdint> diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index 9d0ad369a2..d1e92ce607 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,8 +7,8 @@ #include <consensus/validation.h> #include <core_io.h> #include <core_memusage.h> -#include <pubkey.h> #include <primitives/block.h> +#include <pubkey.h> #include <streams.h> #include <test/fuzz/fuzz.h> #include <validation.h> diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index d1112f8e62..50036ce5bd 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -25,7 +25,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; while (fuzzed_data_provider.remaining_bytes() > 0) { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 6)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 4)) { case 0: { const std::vector<unsigned char> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); (void)bloom_filter.contains(b); @@ -56,13 +56,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(present); break; } - case 3: - bloom_filter.clear(); - break; - case 4: - bloom_filter.reset(fuzzed_data_provider.ConsumeIntegral<unsigned int>()); - break; - case 5: { + case 3: { const Optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mut_tx) { break; @@ -71,7 +65,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)bloom_filter.IsRelevantAndUpdate(tx); break; } - case 6: + case 4: bloom_filter.UpdateEmptyFull(); break; } diff --git a/src/test/fuzz/checkqueue.cpp b/src/test/fuzz/checkqueue.cpp new file mode 100644 index 0000000000..2ed097b827 --- /dev/null +++ b/src/test/fuzz/checkqueue.cpp @@ -0,0 +1,65 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <checkqueue.h> +#include <optional.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +namespace { +struct DumbCheck { + const bool result = false; + + DumbCheck() = default; + + explicit DumbCheck(const bool _result) : result(_result) + { + } + + bool operator()() const + { + return result; + } + + void swap(DumbCheck& x) + { + } +}; +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const unsigned int batch_size = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 1024); + CCheckQueue<DumbCheck> check_queue_1{batch_size}; + CCheckQueue<DumbCheck> check_queue_2{batch_size}; + std::vector<DumbCheck> checks_1; + std::vector<DumbCheck> checks_2; + const int size = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1024); + for (int i = 0; i < size; ++i) { + const bool result = fuzzed_data_provider.ConsumeBool(); + checks_1.emplace_back(result); + checks_2.emplace_back(result); + } + if (fuzzed_data_provider.ConsumeBool()) { + check_queue_1.Add(checks_1); + } + if (fuzzed_data_provider.ConsumeBool()) { + (void)check_queue_1.Wait(); + } + + CCheckQueueControl<DumbCheck> check_queue_control{&check_queue_2}; + if (fuzzed_data_provider.ConsumeBool()) { + check_queue_control.Add(checks_2); + } + if (fuzzed_data_provider.ConsumeBool()) { + (void)check_queue_control.Wait(); + } +} diff --git a/src/test/fuzz/cuckoocache.cpp b/src/test/fuzz/cuckoocache.cpp new file mode 100644 index 0000000000..f674efe1b1 --- /dev/null +++ b/src/test/fuzz/cuckoocache.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <cuckoocache.h> +#include <optional.h> +#include <script/sigcache.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> + +#include <cstdint> +#include <string> +#include <vector> + +namespace { +FuzzedDataProvider* fuzzed_data_provider_ptr = nullptr; + +struct RandomHasher { + template <uint8_t> + uint32_t operator()(const bool& /* unused */) const + { + assert(fuzzed_data_provider_ptr != nullptr); + return fuzzed_data_provider_ptr->ConsumeIntegral<uint32_t>(); + } +}; +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + fuzzed_data_provider_ptr = &fuzzed_data_provider; + CuckooCache::cache<bool, RandomHasher> cuckoo_cache{}; + if (fuzzed_data_provider.ConsumeBool()) { + const size_t megabytes = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 16); + cuckoo_cache.setup_bytes(megabytes << 20); + } else { + cuckoo_cache.setup(fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(0, 4096)); + } + while (fuzzed_data_provider.ConsumeBool()) { + if (fuzzed_data_provider.ConsumeBool()) { + cuckoo_cache.insert(fuzzed_data_provider.ConsumeBool()); + } else { + cuckoo_cache.contains(fuzzed_data_provider.ConsumeBool(), fuzzed_data_provider.ConsumeBool()); + } + } + fuzzed_data_provider_ptr = nullptr; +} diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index 1e0abe94f8..001758ffdb 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 964fc85302..54793c890f 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/eval_script.cpp b/src/test/fuzz/eval_script.cpp index 6a1b037630..c556599db3 100644 --- a/src/test/fuzz/eval_script.cpp +++ b/src/test/fuzz/eval_script.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <pubkey.h> #include <script/interpreter.h> -#include <test/fuzz/fuzz.h> #include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> #include <util/memory.h> #include <limits> diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp new file mode 100644 index 0000000000..090994263e --- /dev/null +++ b/src/test/fuzz/fees.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <amount.h> +#include <optional.h> +#include <policy/fees.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const CFeeRate minimal_incremental_fee{ConsumeMoney(fuzzed_data_provider)}; + FeeFilterRounder fee_filter_rounder{minimal_incremental_fee}; + while (fuzzed_data_provider.ConsumeBool()) { + const CAmount current_minimum_fee = ConsumeMoney(fuzzed_data_provider); + const CAmount rounded_fee = fee_filter_rounder.round(current_minimum_fee); + assert(MoneyRange(rounded_fee)); + } +} diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index a085e36911..6e2188fe86 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/hex.cpp b/src/test/fuzz/hex.cpp index 3bbf0084c2..5fed17c17c 100644 --- a/src/test/fuzz/hex.cpp +++ b/src/test/fuzz/hex.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <core_io.h> -#include <pubkey.h> #include <primitives/block.h> +#include <pubkey.h> #include <rpc/util.h> #include <test/fuzz/fuzz.h> #include <uint256.h> diff --git a/src/test/fuzz/http_request.cpp b/src/test/fuzz/http_request.cpp new file mode 100644 index 0000000000..ebf89749e9 --- /dev/null +++ b/src/test/fuzz/http_request.cpp @@ -0,0 +1,73 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <httpserver.h> +#include <netaddress.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <event2/buffer.h> +#include <event2/event.h> +#include <event2/http.h> +#include <event2/http_struct.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +// workaround for libevent versions before 2.1.1, +// when internal functions didn't have underscores at the end +#if LIBEVENT_VERSION_NUMBER < 0x02010100 +extern "C" int evhttp_parse_firstline(struct evhttp_request*, struct evbuffer*); +extern "C" int evhttp_parse_headers(struct evhttp_request*, struct evbuffer*); +inline int evhttp_parse_firstline_(struct evhttp_request* r, struct evbuffer* b) +{ + return evhttp_parse_firstline(r, b); +} +inline int evhttp_parse_headers_(struct evhttp_request* r, struct evbuffer* b) +{ + return evhttp_parse_headers(r, b); +} +#else +extern "C" int evhttp_parse_firstline_(struct evhttp_request*, struct evbuffer*); +extern "C" int evhttp_parse_headers_(struct evhttp_request*, struct evbuffer*); +#endif + +std::string RequestMethodString(HTTPRequest::RequestMethod m); + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + evhttp_request* evreq = evhttp_request_new(nullptr, nullptr); + assert(evreq != nullptr); + evreq->kind = EVHTTP_REQUEST; + evbuffer* evbuf = evbuffer_new(); + assert(evbuf != nullptr); + const std::vector<uint8_t> http_buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, 4096); + evbuffer_add(evbuf, http_buffer.data(), http_buffer.size()); + if (evhttp_parse_firstline_(evreq, evbuf) != 1 || evhttp_parse_headers_(evreq, evbuf) != 1) { + evbuffer_free(evbuf); + evhttp_request_free(evreq); + return; + } + + HTTPRequest http_request{evreq, true}; + const HTTPRequest::RequestMethod request_method = http_request.GetRequestMethod(); + (void)RequestMethodString(request_method); + (void)http_request.GetURI(); + (void)http_request.GetHeader("Host"); + const std::string header = fuzzed_data_provider.ConsumeRandomLengthString(16); + (void)http_request.GetHeader(header); + (void)http_request.WriteHeader(header, fuzzed_data_provider.ConsumeRandomLengthString(16)); + (void)http_request.GetHeader(header); + const std::string body = http_request.ReadBody(); + assert(body.empty()); + const CService service = http_request.GetPeer(); + assert(service.ToString() == "[::]:0"); + + evbuffer_free(evbuf); + evhttp_request_free(evreq); +} diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index fff2fabd17..9dbf0fcc90 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,6 +23,7 @@ #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> #include <time.h> #include <uint256.h> #include <util/moneystr.h> @@ -35,6 +36,7 @@ #include <cassert> #include <chrono> #include <limits> +#include <set> #include <vector> void initialize() @@ -90,8 +92,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) } (void)GetSizeOfCompactSize(u64); (void)GetSpecialScriptSize(u32); - // (void)GetVirtualTransactionSize(i64, i64); // function defined only for a subset of int64_t inputs - // (void)GetVirtualTransactionSize(i64, i64, u32); // function defined only for a subset of int64_t/uint32_t inputs + if (!MultiplicationOverflow(i64, static_cast<int64_t>(::nBytesPerSigOp)) && !AdditionOverflow(i64 * ::nBytesPerSigOp, static_cast<int64_t>(4))) { + (void)GetVirtualTransactionSize(i64, i64); + } + if (!MultiplicationOverflow(i64, static_cast<int64_t>(u32)) && !AdditionOverflow(i64, static_cast<int64_t>(4)) && !AdditionOverflow(i64 * u32, static_cast<int64_t>(4))) { + (void)GetVirtualTransactionSize(i64, i64, u32); + } (void)HexDigit(ch); (void)MoneyRange(i64); (void)ToString(i64); @@ -109,6 +115,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)memusage::DynamicUsage(u8); const unsigned char uch = static_cast<unsigned char>(u8); (void)memusage::DynamicUsage(uch); + { + const std::set<int64_t> i64s{i64, static_cast<int64_t>(u64)}; + const size_t dynamic_usage = memusage::DynamicUsage(i64s); + const size_t incremental_dynamic_usage = memusage::IncrementalDynamicUsage(i64s); + assert(dynamic_usage == incremental_dynamic_usage * i64s.size()); + } (void)MillisToTimeval(i64); const double d = ser_uint64_to_double(u64); assert(ser_double_to_uint64(d) == u64); diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 09580c0c52..3597f51e51 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -17,8 +17,7 @@ namespace { const std::string locale_identifiers[] = { - "C", "C.UTF-8", "aa_DJ", "aa_DJ.ISO-8859-1", "aa_DJ.UTF-8", "aa_ER", "aa_ER.UTF-8", "aa_ET", "aa_ET.UTF-8", "af_ZA", "af_ZA.ISO-8859-1", "af_ZA.UTF-8", "agr_PE", "agr_PE.UTF-8", "ak_GH", "ak_GH.UTF-8", "am_ET", "am_ET.UTF-8", "an_ES", "an_ES.ISO-8859-15", "an_ES.UTF-8", "anp_IN", "anp_IN.UTF-8", "ar_AE", "ar_AE.ISO-8859-6", "ar_AE.UTF-8", "ar_BH", "ar_BH.ISO-8859-6", "ar_BH.UTF-8", "ar_DZ", "ar_DZ.ISO-8859-6", "ar_DZ.UTF-8", "ar_EG", "ar_EG.ISO-8859-6", "ar_EG.UTF-8", "ar_IN", "ar_IN.UTF-8", "ar_IQ", "ar_IQ.ISO-8859-6", "ar_IQ.UTF-8", "ar_JO", "ar_JO.ISO-8859-6", "ar_JO.UTF-8", "ar_KW", "ar_KW.ISO-8859-6", "ar_KW.UTF-8", "ar_LB", "ar_LB.ISO-8859-6", "ar_LB.UTF-8", "ar_LY", "ar_LY.ISO-8859-6", "ar_LY.UTF-8", "ar_MA", "ar_MA.ISO-8859-6", "ar_MA.UTF-8", "ar_OM", "ar_OM.ISO-8859-6", "ar_OM.UTF-8", "ar_QA", "ar_QA.ISO-8859-6", "ar_QA.UTF-8", "ar_SA", "ar_SA.ISO-8859-6", "ar_SA.UTF-8", "ar_SD", "ar_SD.ISO-8859-6", "ar_SD.UTF-8", "ar_SS", "ar_SS.UTF-8", "ar_SY", "ar_SY.ISO-8859-6", "ar_SY.UTF-8", "ar_TN", "ar_TN.ISO-8859-6", "ar_TN.UTF-8", "ar_YE", "ar_YE.ISO-8859-6", "ar_YE.UTF-8", "as_IN", "as_IN.UTF-8", "ast_ES", "ast_ES.ISO-8859-15", "ast_ES.UTF-8", "ayc_PE", "ayc_PE.UTF-8", "az_AZ", "az_AZ.UTF-8", "az_IR", "az_IR.UTF-8", "be_BY", "be_BY.CP1251", "be_BY.UTF-8", "bem_ZM", "bem_ZM.UTF-8", "ber_DZ", "ber_DZ.UTF-8", "ber_MA", "ber_MA.UTF-8", "bg_BG", "bg_BG.CP1251", "bg_BG.UTF-8", "bho_IN", "bho_IN.UTF-8", "bho_NP", "bho_NP.UTF-8", "bi_VU", "bi_VU.UTF-8", "bn_BD", "bn_BD.UTF-8", "bn_IN", "bn_IN.UTF-8", "bo_CN", "bo_CN.UTF-8", "bo_IN", "bo_IN.UTF-8", "br_FR", "br_FR.ISO-8859-1", "br_FR.UTF-8", "brx_IN", "brx_IN.UTF-8", "bs_BA", "bs_BA.ISO-8859-2", "bs_BA.UTF-8", "byn_ER", "byn_ER.UTF-8", "ca_AD", "ca_AD.ISO-8859-15", "ca_AD.UTF-8", "ca_ES", "ca_ES.ISO-8859-1", "ca_ES.UTF-8", "ca_FR", "ca_FR.ISO-8859-15", "ca_FR.UTF-8", "ca_IT", "ca_IT.ISO-8859-15", "ca_IT.UTF-8", "ce_RU", "ce_RU.UTF-8", "chr_US", "chr_US.UTF-8", "ckb_IQ", "ckb_IQ.UTF-8", "cmn_TW", "cmn_TW.UTF-8", "crh_UA", "crh_UA.UTF-8", "csb_PL", "csb_PL.UTF-8", "cs_CZ", "cs_CZ.ISO-8859-2", "cs_CZ.UTF-8", "cv_RU", "cv_RU.UTF-8", "cy_GB", "cy_GB.ISO-8859-14", "cy_GB.UTF-8", "da_DK", "da_DK.ISO-8859-1", "da_DK.UTF-8", "de_AT", "de_AT.ISO-8859-1", "de_AT.UTF-8", "de_BE", "de_BE.ISO-8859-1", "de_BE.UTF-8", "de_CH", "de_CH.ISO-8859-1", "de_CH.UTF-8", "de_DE", "de_DE.ISO-8859-1", "de_DE.UTF-8", "de_IT", "de_IT.ISO-8859-1", "de_IT.UTF-8", "de_LU", "de_LU.ISO-8859-1", "de_LU.UTF-8", "doi_IN", "doi_IN.UTF-8", "dv_MV", "dv_MV.UTF-8", "dz_BT", "dz_BT.UTF-8", "el_CY", "el_CY.ISO-8859-7", "el_CY.UTF-8", "el_GR", "el_GR.ISO-8859-7", "el_GR.UTF-8", "en_AG", "en_AG.UTF-8", "en_AU", "en_AU.ISO-8859-1", "en_AU.UTF-8", "en_BW", "en_BW.ISO-8859-1", "en_BW.UTF-8", "en_CA", "en_CA.ISO-8859-1", "en_CA.UTF-8", "en_DK", "en_DK.ISO-8859-1", "en_DK.ISO-8859-15", "en_DK.UTF-8", "en_GB", "en_GB.ISO-8859-1", "en_GB.ISO-8859-15", "en_GB.UTF-8", "en_HK", "en_HK.ISO-8859-1", "en_HK.UTF-8", "en_IE", "en_IE.ISO-8859-1", "en_IE.UTF-8", "en_IL", "en_IL.UTF-8", "en_IN", "en_IN.UTF-8", "en_NG", "en_NG.UTF-8", "en_NZ", "en_NZ.ISO-8859-1", "en_NZ.UTF-8", "en_PH", "en_PH.ISO-8859-1", "en_PH.UTF-8", "en_SG", "en_SG.ISO-8859-1", "en_SG.UTF-8", "en_US", "en_US.ISO-8859-1", "en_US.ISO-8859-15", "en_US.UTF-8", "en_ZA", "en_ZA.ISO-8859-1", "en_ZA.UTF-8", "en_ZM", "en_ZM.UTF-8", "en_ZW", "en_ZW.ISO-8859-1", "en_ZW.UTF-8", "es_AR", "es_AR.ISO-8859-1", "es_AR.UTF-8", "es_BO", "es_BO.ISO-8859-1", "es_BO.UTF-8", "es_CL", "es_CL.ISO-8859-1", "es_CL.UTF-8", "es_CO", "es_CO.ISO-8859-1", "es_CO.UTF-8", "es_CR", "es_CR.ISO-8859-1", "es_CR.UTF-8", "es_CU", "es_CU.UTF-8", "es_DO", "es_DO.ISO-8859-1", "es_DO.UTF-8", "es_EC", "es_EC.ISO-8859-1", "es_EC.UTF-8", "es_ES", "es_ES.ISO-8859-1", "es_ES.UTF-8", "es_GT", "es_GT.ISO-8859-1", "es_GT.UTF-8", "es_HN", "es_HN.ISO-8859-1", "es_HN.UTF-8", "es_MX", "es_MX.ISO-8859-1", "es_MX.UTF-8", "es_NI", "es_NI.ISO-8859-1", "es_NI.UTF-8", "es_PA", "es_PA.ISO-8859-1", "es_PA.UTF-8", "es_PE", "es_PE.ISO-8859-1", "es_PE.UTF-8", "es_PR", "es_PR.ISO-8859-1", "es_PR.UTF-8", "es_PY", "es_PY.ISO-8859-1", "es_PY.UTF-8", "es_SV", "es_SV.ISO-8859-1", "es_SV.UTF-8", "es_US", "es_US.ISO-8859-1", "es_US.UTF-8", "es_UY", "es_UY.ISO-8859-1", "es_UY.UTF-8", "es_VE", "es_VE.ISO-8859-1", "es_VE.UTF-8", "et_EE", "et_EE.ISO-8859-1", "et_EE.ISO-8859-15", "et_EE.UTF-8", "eu_ES", "eu_ES.ISO-8859-1", "eu_ES.UTF-8", "eu_FR", "eu_FR.ISO-8859-1", "eu_FR.UTF-8", "fa_IR", "fa_IR.UTF-8", "ff_SN", "ff_SN.UTF-8", "fi_FI", "fi_FI.ISO-8859-1", "fi_FI.UTF-8", "fil_PH", "fil_PH.UTF-8", "fo_FO", "fo_FO.ISO-8859-1", "fo_FO.UTF-8", "fr_BE", "fr_BE.ISO-8859-1", "fr_BE.UTF-8", "fr_CA", "fr_CA.ISO-8859-1", "fr_CA.UTF-8", "fr_CH", "fr_CH.ISO-8859-1", "fr_CH.UTF-8", "fr_FR", "fr_FR.ISO-8859-1", "fr_FR.UTF-8", "fr_LU", "fr_LU.ISO-8859-1", "fr_LU.UTF-8", "fur_IT", "fur_IT.UTF-8", "fy_DE", "fy_DE.UTF-8", "fy_NL", "fy_NL.UTF-8", "ga_IE", "ga_IE.ISO-8859-1", "ga_IE.UTF-8", "gd_GB", "gd_GB.ISO-8859-15", "gd_GB.UTF-8", "gez_ER", "gez_ER.UTF-8", "gez_ET", "gez_ET.UTF-8", "gl_ES", "gl_ES.ISO-8859-1", "gl_ES.UTF-8", "gu_IN", "gu_IN.UTF-8", "gv_GB", "gv_GB.ISO-8859-1", "gv_GB.UTF-8", "hak_TW", "hak_TW.UTF-8", "ha_NG", "ha_NG.UTF-8", "he_IL", "he_IL.ISO-8859-8", "he_IL.UTF-8", "hif_FJ", "hif_FJ.UTF-8", "hi_IN", "hi_IN.UTF-8", "hne_IN", "hne_IN.UTF-8", "hr_HR", "hr_HR.ISO-8859-2", "hr_HR.UTF-8", "hsb_DE", "hsb_DE.ISO-8859-2", "hsb_DE.UTF-8", "ht_HT", "ht_HT.UTF-8", "hu_HU", "hu_HU.ISO-8859-2", "hu_HU.UTF-8", "hy_AM", "hy_AM.ARMSCII-8", "hy_AM.UTF-8", "ia_FR", "ia_FR.UTF-8", "id_ID", "id_ID.ISO-8859-1", "id_ID.UTF-8", "ig_NG", "ig_NG.UTF-8", "ik_CA", "ik_CA.UTF-8", "is_IS", "is_IS.ISO-8859-1", "is_IS.UTF-8", "it_CH", "it_CH.ISO-8859-1", "it_CH.UTF-8", "it_IT", "it_IT.ISO-8859-1", "it_IT.UTF-8", "iu_CA", "iu_CA.UTF-8", "kab_DZ", "kab_DZ.UTF-8", "ka_GE", "ka_GE.GEORGIAN-PS", "ka_GE.UTF-8", "kk_KZ", "kk_KZ.PT154", "kk_KZ.RK1048", "kk_KZ.UTF-8", "kl_GL", "kl_GL.ISO-8859-1", "kl_GL.UTF-8", "km_KH", "km_KH.UTF-8", "kn_IN", "kn_IN.UTF-8", "kok_IN", "kok_IN.UTF-8", "ks_IN", "ks_IN.UTF-8", "ku_TR", "ku_TR.ISO-8859-9", "ku_TR.UTF-8", "kw_GB", "kw_GB.ISO-8859-1", "kw_GB.UTF-8", "ky_KG", "ky_KG.UTF-8", "lb_LU", "lb_LU.UTF-8", "lg_UG", "lg_UG.ISO-8859-10", "lg_UG.UTF-8", "li_BE", "li_BE.UTF-8", "lij_IT", "lij_IT.UTF-8", "li_NL", "li_NL.UTF-8", "ln_CD", "ln_CD.UTF-8", "lo_LA", "lo_LA.UTF-8", "lt_LT", "lt_LT.ISO-8859-13", "lt_LT.UTF-8", "lv_LV", "lv_LV.ISO-8859-13", "lv_LV.UTF-8", "lzh_TW", "lzh_TW.UTF-8", "mag_IN", "mag_IN.UTF-8", "mai_IN", "mai_IN.UTF-8", "mai_NP", "mai_NP.UTF-8", "mfe_MU", "mfe_MU.UTF-8", "mg_MG", "mg_MG.ISO-8859-15", "mg_MG.UTF-8", "mhr_RU", "mhr_RU.UTF-8", "mi_NZ", "mi_NZ.ISO-8859-13", "mi_NZ.UTF-8", "miq_NI", "miq_NI.UTF-8", "mjw_IN", "mjw_IN.UTF-8", "mk_MK", "mk_MK.ISO-8859-5", "mk_MK.UTF-8", "ml_IN", "ml_IN.UTF-8", "mni_IN", "mni_IN.UTF-8", "mn_MN", "mn_MN.UTF-8", "mr_IN", "mr_IN.UTF-8", "ms_MY", "ms_MY.ISO-8859-1", "ms_MY.UTF-8", "mt_MT", "mt_MT.ISO-8859-3", "mt_MT.UTF-8", "my_MM", "my_MM.UTF-8", "nan_TW", "nan_TW.UTF-8", "nb_NO", "nb_NO.ISO-8859-1", "nb_NO.UTF-8", "nds_DE", "nds_DE.UTF-8", "nds_NL", "nds_NL.UTF-8", "ne_NP", "ne_NP.UTF-8", "nhn_MX", "nhn_MX.UTF-8", "niu_NU", "niu_NU.UTF-8", "niu_NZ", "niu_NZ.UTF-8", "nl_AW", "nl_AW.UTF-8", "nl_BE", "nl_BE.ISO-8859-1", "nl_BE.UTF-8", "nl_NL", "nl_NL.ISO-8859-1", "nl_NL.UTF-8", "nn_NO", "nn_NO.ISO-8859-1", "nn_NO.UTF-8", "nr_ZA", "nr_ZA.UTF-8", "nso_ZA", "nso_ZA.UTF-8", "oc_FR", "oc_FR.ISO-8859-1", "oc_FR.UTF-8", "om_ET", "om_ET.UTF-8", "om_KE", "om_KE.ISO-8859-1", "om_KE.UTF-8", "or_IN", "or_IN.UTF-8", "os_RU", "os_RU.UTF-8", "pa_IN", "pa_IN.UTF-8", "pap_AW", "pap_AW.UTF-8", "pap_CW", "pap_CW.UTF-8", "pa_PK", "pa_PK.UTF-8", "pl_PL", "pl_PL.ISO-8859-2", "pl_PL.UTF-8", "ps_AF", "ps_AF.UTF-8", "pt_BR", "pt_BR.ISO-8859-1", "pt_BR.UTF-8", "pt_PT", "pt_PT.ISO-8859-1", "pt_PT.UTF-8", "quz_PE", "quz_PE.UTF-8", "raj_IN", "raj_IN.UTF-8", "ro_RO", "ro_RO.ISO-8859-2", "ro_RO.UTF-8", "ru_RU", "ru_RU.CP1251", "ru_RU.ISO-8859-5", "ru_RU.KOI8-R", "ru_RU.UTF-8", "ru_UA", "ru_UA.KOI8-U", "ru_UA.UTF-8", "rw_RW", "rw_RW.UTF-8", "sa_IN", "sa_IN.UTF-8", "sat_IN", "sat_IN.UTF-8", "sc_IT", "sc_IT.UTF-8", "sd_IN", "sd_IN.UTF-8", "sd_PK", "sd_PK.UTF-8", "se_NO", "se_NO.UTF-8", "sgs_LT", "sgs_LT.UTF-8", "shn_MM", "shn_MM.UTF-8", "shs_CA", "shs_CA.UTF-8", "sid_ET", "sid_ET.UTF-8", "si_LK", "si_LK.UTF-8", "sk_SK", "sk_SK.ISO-8859-2", "sk_SK.UTF-8", "sl_SI", "sl_SI.ISO-8859-2", "sl_SI.UTF-8", "sm_WS", "sm_WS.UTF-8", "so_DJ", "so_DJ.ISO-8859-1", "so_DJ.UTF-8", "so_ET", "so_ET.UTF-8", "so_KE", "so_KE.ISO-8859-1", "so_KE.UTF-8", "so_SO", "so_SO.ISO-8859-1", "so_SO.UTF-8", "sq_AL", "sq_AL.ISO-8859-1", "sq_AL.UTF-8", "sq_MK", "sq_MK.UTF-8", "sr_ME", "sr_ME.UTF-8", "sr_RS", "sr_RS.UTF-8", "ss_ZA", "ss_ZA.UTF-8", "st_ZA", "st_ZA.ISO-8859-1", "st_ZA.UTF-8", "sv_FI", "sv_FI.ISO-8859-1", "sv_FI.UTF-8", "sv_SE", "sv_SE.ISO-8859-1", "sv_SE.ISO-8859-15", "sv_SE.UTF-8", "sw_KE", "sw_KE.UTF-8", "sw_TZ", "sw_TZ.UTF-8", "szl_PL", "szl_PL.UTF-8", "ta_IN", "ta_IN.UTF-8", "ta_LK", "ta_LK.UTF-8", "te_IN", "te_IN.UTF-8", "tg_TJ", "tg_TJ.KOI8-T", "tg_TJ.UTF-8", "the_NP", "the_NP.UTF-8", "th_TH", "th_TH.TIS-620", "th_TH.UTF-8", "ti_ER", "ti_ER.UTF-8", "ti_ET", "ti_ET.UTF-8", "tig_ER", "tig_ER.UTF-8", "tk_TM", "tk_TM.UTF-8", "tl_PH", "tl_PH.ISO-8859-1", "tl_PH.UTF-8", "tn_ZA", "tn_ZA.UTF-8", "to_TO", "to_TO.UTF-8", "tpi_PG", "tpi_PG.UTF-8", "tr_CY", "tr_CY.ISO-8859-9", "tr_CY.UTF-8", "tr_TR", "tr_TR.ISO-8859-9", "tr_TR.UTF-8", "ts_ZA", "ts_ZA.UTF-8", "tt_RU", "tt_RU.UTF-8", "ug_CN", "ug_CN.UTF-8", "uk_UA", "uk_UA.KOI8-U", "uk_UA.UTF-8", "unm_US", "unm_US.UTF-8", "ur_IN", "ur_IN.UTF-8", "ur_PK", "ur_PK.UTF-8", "uz_UZ", "uz_UZ.ISO-8859-1", "uz_UZ.UTF-8", "ve_ZA", "ve_ZA.UTF-8", "vi_VN", "vi_VN.UTF-8", "wa_BE", "wa_BE.ISO-8859-1", "wa_BE.UTF-8", "wae_CH", "wae_CH.UTF-8", "wal_ET", "wal_ET.UTF-8", "wo_SN", "wo_SN.UTF-8", "xh_ZA", "xh_ZA.ISO-8859-1", "xh_ZA.UTF-8", "yi_US", "yi_US.CP1255", "yi_US.UTF-8", "yo_NG", "yo_NG.UTF-8", "yue_HK", "yue_HK.UTF-8", "yuw_PG", "yuw_PG.UTF-8", "zh_CN", "zh_CN.GB18030", "zh_CN.GB2312", "zh_CN.GBK", "zh_CN.UTF-8", "zh_HK", "zh_HK.BIG5-HKSCS", "zh_HK.UTF-8", "zh_SG", "zh_SG.GB2312", "zh_SG.GBK", "zh_SG.UTF-8", "zh_TW", "zh_TW.BIG5", "zh_TW.EUC-TW", "zh_TW.UTF-8", "zu_ZA", "zu_ZA.ISO-8859-1", "zu_ZA.UTF-8" -}; + "C", "C.UTF-8", "aa_DJ", "aa_DJ.ISO-8859-1", "aa_DJ.UTF-8", "aa_ER", "aa_ER.UTF-8", "aa_ET", "aa_ET.UTF-8", "af_ZA", "af_ZA.ISO-8859-1", "af_ZA.UTF-8", "agr_PE", "agr_PE.UTF-8", "ak_GH", "ak_GH.UTF-8", "am_ET", "am_ET.UTF-8", "an_ES", "an_ES.ISO-8859-15", "an_ES.UTF-8", "anp_IN", "anp_IN.UTF-8", "ar_AE", "ar_AE.ISO-8859-6", "ar_AE.UTF-8", "ar_BH", "ar_BH.ISO-8859-6", "ar_BH.UTF-8", "ar_DZ", "ar_DZ.ISO-8859-6", "ar_DZ.UTF-8", "ar_EG", "ar_EG.ISO-8859-6", "ar_EG.UTF-8", "ar_IN", "ar_IN.UTF-8", "ar_IQ", "ar_IQ.ISO-8859-6", "ar_IQ.UTF-8", "ar_JO", "ar_JO.ISO-8859-6", "ar_JO.UTF-8", "ar_KW", "ar_KW.ISO-8859-6", "ar_KW.UTF-8", "ar_LB", "ar_LB.ISO-8859-6", "ar_LB.UTF-8", "ar_LY", "ar_LY.ISO-8859-6", "ar_LY.UTF-8", "ar_MA", "ar_MA.ISO-8859-6", "ar_MA.UTF-8", "ar_OM", "ar_OM.ISO-8859-6", "ar_OM.UTF-8", "ar_QA", "ar_QA.ISO-8859-6", "ar_QA.UTF-8", "ar_SA", "ar_SA.ISO-8859-6", "ar_SA.UTF-8", "ar_SD", "ar_SD.ISO-8859-6", "ar_SD.UTF-8", "ar_SS", "ar_SS.UTF-8", "ar_SY", "ar_SY.ISO-8859-6", "ar_SY.UTF-8", "ar_TN", "ar_TN.ISO-8859-6", "ar_TN.UTF-8", "ar_YE", "ar_YE.ISO-8859-6", "ar_YE.UTF-8", "as_IN", "as_IN.UTF-8", "ast_ES", "ast_ES.ISO-8859-15", "ast_ES.UTF-8", "ayc_PE", "ayc_PE.UTF-8", "az_AZ", "az_AZ.UTF-8", "az_IR", "az_IR.UTF-8", "be_BY", "be_BY.CP1251", "be_BY.UTF-8", "bem_ZM", "bem_ZM.UTF-8", "ber_DZ", "ber_DZ.UTF-8", "ber_MA", "ber_MA.UTF-8", "bg_BG", "bg_BG.CP1251", "bg_BG.UTF-8", "bho_IN", "bho_IN.UTF-8", "bho_NP", "bho_NP.UTF-8", "bi_VU", "bi_VU.UTF-8", "bn_BD", "bn_BD.UTF-8", "bn_IN", "bn_IN.UTF-8", "bo_CN", "bo_CN.UTF-8", "bo_IN", "bo_IN.UTF-8", "br_FR", "br_FR.ISO-8859-1", "br_FR.UTF-8", "brx_IN", "brx_IN.UTF-8", "bs_BA", "bs_BA.ISO-8859-2", "bs_BA.UTF-8", "byn_ER", "byn_ER.UTF-8", "ca_AD", "ca_AD.ISO-8859-15", "ca_AD.UTF-8", "ca_ES", "ca_ES.ISO-8859-1", "ca_ES.UTF-8", "ca_FR", "ca_FR.ISO-8859-15", "ca_FR.UTF-8", "ca_IT", "ca_IT.ISO-8859-15", "ca_IT.UTF-8", "ce_RU", "ce_RU.UTF-8", "chr_US", "chr_US.UTF-8", "ckb_IQ", "ckb_IQ.UTF-8", "cmn_TW", "cmn_TW.UTF-8", "crh_UA", "crh_UA.UTF-8", "csb_PL", "csb_PL.UTF-8", "cs_CZ", "cs_CZ.ISO-8859-2", "cs_CZ.UTF-8", "cv_RU", "cv_RU.UTF-8", "cy_GB", "cy_GB.ISO-8859-14", "cy_GB.UTF-8", "da_DK", "da_DK.ISO-8859-1", "da_DK.UTF-8", "de_AT", "de_AT.ISO-8859-1", "de_AT.UTF-8", "de_BE", "de_BE.ISO-8859-1", "de_BE.UTF-8", "de_CH", "de_CH.ISO-8859-1", "de_CH.UTF-8", "de_DE", "de_DE.ISO-8859-1", "de_DE.UTF-8", "de_IT", "de_IT.ISO-8859-1", "de_IT.UTF-8", "de_LU", "de_LU.ISO-8859-1", "de_LU.UTF-8", "doi_IN", "doi_IN.UTF-8", "dv_MV", "dv_MV.UTF-8", "dz_BT", "dz_BT.UTF-8", "el_CY", "el_CY.ISO-8859-7", "el_CY.UTF-8", "el_GR", "el_GR.ISO-8859-7", "el_GR.UTF-8", "en_AG", "en_AG.UTF-8", "en_AU", "en_AU.ISO-8859-1", "en_AU.UTF-8", "en_BW", "en_BW.ISO-8859-1", "en_BW.UTF-8", "en_CA", "en_CA.ISO-8859-1", "en_CA.UTF-8", "en_DK", "en_DK.ISO-8859-1", "en_DK.ISO-8859-15", "en_DK.UTF-8", "en_GB", "en_GB.ISO-8859-1", "en_GB.ISO-8859-15", "en_GB.UTF-8", "en_HK", "en_HK.ISO-8859-1", "en_HK.UTF-8", "en_IE", "en_IE.ISO-8859-1", "en_IE.UTF-8", "en_IL", "en_IL.UTF-8", "en_IN", "en_IN.UTF-8", "en_NG", "en_NG.UTF-8", "en_NZ", "en_NZ.ISO-8859-1", "en_NZ.UTF-8", "en_PH", "en_PH.ISO-8859-1", "en_PH.UTF-8", "en_SG", "en_SG.ISO-8859-1", "en_SG.UTF-8", "en_US", "en_US.ISO-8859-1", "en_US.ISO-8859-15", "en_US.UTF-8", "en_ZA", "en_ZA.ISO-8859-1", "en_ZA.UTF-8", "en_ZM", "en_ZM.UTF-8", "en_ZW", "en_ZW.ISO-8859-1", "en_ZW.UTF-8", "es_AR", "es_AR.ISO-8859-1", "es_AR.UTF-8", "es_BO", "es_BO.ISO-8859-1", "es_BO.UTF-8", "es_CL", "es_CL.ISO-8859-1", "es_CL.UTF-8", "es_CO", "es_CO.ISO-8859-1", "es_CO.UTF-8", "es_CR", "es_CR.ISO-8859-1", "es_CR.UTF-8", "es_CU", "es_CU.UTF-8", "es_DO", "es_DO.ISO-8859-1", "es_DO.UTF-8", "es_EC", "es_EC.ISO-8859-1", "es_EC.UTF-8", "es_ES", "es_ES.ISO-8859-1", "es_ES.UTF-8", "es_GT", "es_GT.ISO-8859-1", "es_GT.UTF-8", "es_HN", "es_HN.ISO-8859-1", "es_HN.UTF-8", "es_MX", "es_MX.ISO-8859-1", "es_MX.UTF-8", "es_NI", "es_NI.ISO-8859-1", "es_NI.UTF-8", "es_PA", "es_PA.ISO-8859-1", "es_PA.UTF-8", "es_PE", "es_PE.ISO-8859-1", "es_PE.UTF-8", "es_PR", "es_PR.ISO-8859-1", "es_PR.UTF-8", "es_PY", "es_PY.ISO-8859-1", "es_PY.UTF-8", "es_SV", "es_SV.ISO-8859-1", "es_SV.UTF-8", "es_US", "es_US.ISO-8859-1", "es_US.UTF-8", "es_UY", "es_UY.ISO-8859-1", "es_UY.UTF-8", "es_VE", "es_VE.ISO-8859-1", "es_VE.UTF-8", "et_EE", "et_EE.ISO-8859-1", "et_EE.ISO-8859-15", "et_EE.UTF-8", "eu_ES", "eu_ES.ISO-8859-1", "eu_ES.UTF-8", "eu_FR", "eu_FR.ISO-8859-1", "eu_FR.UTF-8", "fa_IR", "fa_IR.UTF-8", "ff_SN", "ff_SN.UTF-8", "fi_FI", "fi_FI.ISO-8859-1", "fi_FI.UTF-8", "fil_PH", "fil_PH.UTF-8", "fo_FO", "fo_FO.ISO-8859-1", "fo_FO.UTF-8", "fr_BE", "fr_BE.ISO-8859-1", "fr_BE.UTF-8", "fr_CA", "fr_CA.ISO-8859-1", "fr_CA.UTF-8", "fr_CH", "fr_CH.ISO-8859-1", "fr_CH.UTF-8", "fr_FR", "fr_FR.ISO-8859-1", "fr_FR.UTF-8", "fr_LU", "fr_LU.ISO-8859-1", "fr_LU.UTF-8", "fur_IT", "fur_IT.UTF-8", "fy_DE", "fy_DE.UTF-8", "fy_NL", "fy_NL.UTF-8", "ga_IE", "ga_IE.ISO-8859-1", "ga_IE.UTF-8", "gd_GB", "gd_GB.ISO-8859-15", "gd_GB.UTF-8", "gez_ER", "gez_ER.UTF-8", "gez_ET", "gez_ET.UTF-8", "gl_ES", "gl_ES.ISO-8859-1", "gl_ES.UTF-8", "gu_IN", "gu_IN.UTF-8", "gv_GB", "gv_GB.ISO-8859-1", "gv_GB.UTF-8", "hak_TW", "hak_TW.UTF-8", "ha_NG", "ha_NG.UTF-8", "he_IL", "he_IL.ISO-8859-8", "he_IL.UTF-8", "hif_FJ", "hif_FJ.UTF-8", "hi_IN", "hi_IN.UTF-8", "hne_IN", "hne_IN.UTF-8", "hr_HR", "hr_HR.ISO-8859-2", "hr_HR.UTF-8", "hsb_DE", "hsb_DE.ISO-8859-2", "hsb_DE.UTF-8", "ht_HT", "ht_HT.UTF-8", "hu_HU", "hu_HU.ISO-8859-2", "hu_HU.UTF-8", "hy_AM", "hy_AM.ARMSCII-8", "hy_AM.UTF-8", "ia_FR", "ia_FR.UTF-8", "id_ID", "id_ID.ISO-8859-1", "id_ID.UTF-8", "ig_NG", "ig_NG.UTF-8", "ik_CA", "ik_CA.UTF-8", "is_IS", "is_IS.ISO-8859-1", "is_IS.UTF-8", "it_CH", "it_CH.ISO-8859-1", "it_CH.UTF-8", "it_IT", "it_IT.ISO-8859-1", "it_IT.UTF-8", "iu_CA", "iu_CA.UTF-8", "kab_DZ", "kab_DZ.UTF-8", "ka_GE", "ka_GE.GEORGIAN-PS", "ka_GE.UTF-8", "kk_KZ", "kk_KZ.PT154", "kk_KZ.RK1048", "kk_KZ.UTF-8", "kl_GL", "kl_GL.ISO-8859-1", "kl_GL.UTF-8", "km_KH", "km_KH.UTF-8", "kn_IN", "kn_IN.UTF-8", "kok_IN", "kok_IN.UTF-8", "ks_IN", "ks_IN.UTF-8", "ku_TR", "ku_TR.ISO-8859-9", "ku_TR.UTF-8", "kw_GB", "kw_GB.ISO-8859-1", "kw_GB.UTF-8", "ky_KG", "ky_KG.UTF-8", "lb_LU", "lb_LU.UTF-8", "lg_UG", "lg_UG.ISO-8859-10", "lg_UG.UTF-8", "li_BE", "li_BE.UTF-8", "lij_IT", "lij_IT.UTF-8", "li_NL", "li_NL.UTF-8", "ln_CD", "ln_CD.UTF-8", "lo_LA", "lo_LA.UTF-8", "lt_LT", "lt_LT.ISO-8859-13", "lt_LT.UTF-8", "lv_LV", "lv_LV.ISO-8859-13", "lv_LV.UTF-8", "lzh_TW", "lzh_TW.UTF-8", "mag_IN", "mag_IN.UTF-8", "mai_IN", "mai_IN.UTF-8", "mai_NP", "mai_NP.UTF-8", "mfe_MU", "mfe_MU.UTF-8", "mg_MG", "mg_MG.ISO-8859-15", "mg_MG.UTF-8", "mhr_RU", "mhr_RU.UTF-8", "mi_NZ", "mi_NZ.ISO-8859-13", "mi_NZ.UTF-8", "miq_NI", "miq_NI.UTF-8", "mjw_IN", "mjw_IN.UTF-8", "mk_MK", "mk_MK.ISO-8859-5", "mk_MK.UTF-8", "ml_IN", "ml_IN.UTF-8", "mni_IN", "mni_IN.UTF-8", "mn_MN", "mn_MN.UTF-8", "mr_IN", "mr_IN.UTF-8", "ms_MY", "ms_MY.ISO-8859-1", "ms_MY.UTF-8", "mt_MT", "mt_MT.ISO-8859-3", "mt_MT.UTF-8", "my_MM", "my_MM.UTF-8", "nan_TW", "nan_TW.UTF-8", "nb_NO", "nb_NO.ISO-8859-1", "nb_NO.UTF-8", "nds_DE", "nds_DE.UTF-8", "nds_NL", "nds_NL.UTF-8", "ne_NP", "ne_NP.UTF-8", "nhn_MX", "nhn_MX.UTF-8", "niu_NU", "niu_NU.UTF-8", "niu_NZ", "niu_NZ.UTF-8", "nl_AW", "nl_AW.UTF-8", "nl_BE", "nl_BE.ISO-8859-1", "nl_BE.UTF-8", "nl_NL", "nl_NL.ISO-8859-1", "nl_NL.UTF-8", "nn_NO", "nn_NO.ISO-8859-1", "nn_NO.UTF-8", "nr_ZA", "nr_ZA.UTF-8", "nso_ZA", "nso_ZA.UTF-8", "oc_FR", "oc_FR.ISO-8859-1", "oc_FR.UTF-8", "om_ET", "om_ET.UTF-8", "om_KE", "om_KE.ISO-8859-1", "om_KE.UTF-8", "or_IN", "or_IN.UTF-8", "os_RU", "os_RU.UTF-8", "pa_IN", "pa_IN.UTF-8", "pap_AW", "pap_AW.UTF-8", "pap_CW", "pap_CW.UTF-8", "pa_PK", "pa_PK.UTF-8", "pl_PL", "pl_PL.ISO-8859-2", "pl_PL.UTF-8", "ps_AF", "ps_AF.UTF-8", "pt_BR", "pt_BR.ISO-8859-1", "pt_BR.UTF-8", "pt_PT", "pt_PT.ISO-8859-1", "pt_PT.UTF-8", "quz_PE", "quz_PE.UTF-8", "raj_IN", "raj_IN.UTF-8", "ro_RO", "ro_RO.ISO-8859-2", "ro_RO.UTF-8", "ru_RU", "ru_RU.CP1251", "ru_RU.ISO-8859-5", "ru_RU.KOI8-R", "ru_RU.UTF-8", "ru_UA", "ru_UA.KOI8-U", "ru_UA.UTF-8", "rw_RW", "rw_RW.UTF-8", "sa_IN", "sa_IN.UTF-8", "sat_IN", "sat_IN.UTF-8", "sc_IT", "sc_IT.UTF-8", "sd_IN", "sd_IN.UTF-8", "sd_PK", "sd_PK.UTF-8", "se_NO", "se_NO.UTF-8", "sgs_LT", "sgs_LT.UTF-8", "shn_MM", "shn_MM.UTF-8", "shs_CA", "shs_CA.UTF-8", "sid_ET", "sid_ET.UTF-8", "si_LK", "si_LK.UTF-8", "sk_SK", "sk_SK.ISO-8859-2", "sk_SK.UTF-8", "sl_SI", "sl_SI.ISO-8859-2", "sl_SI.UTF-8", "sm_WS", "sm_WS.UTF-8", "so_DJ", "so_DJ.ISO-8859-1", "so_DJ.UTF-8", "so_ET", "so_ET.UTF-8", "so_KE", "so_KE.ISO-8859-1", "so_KE.UTF-8", "so_SO", "so_SO.ISO-8859-1", "so_SO.UTF-8", "sq_AL", "sq_AL.ISO-8859-1", "sq_AL.UTF-8", "sq_MK", "sq_MK.UTF-8", "sr_ME", "sr_ME.UTF-8", "sr_RS", "sr_RS.UTF-8", "ss_ZA", "ss_ZA.UTF-8", "st_ZA", "st_ZA.ISO-8859-1", "st_ZA.UTF-8", "sv_FI", "sv_FI.ISO-8859-1", "sv_FI.UTF-8", "sv_SE", "sv_SE.ISO-8859-1", "sv_SE.ISO-8859-15", "sv_SE.UTF-8", "sw_KE", "sw_KE.UTF-8", "sw_TZ", "sw_TZ.UTF-8", "szl_PL", "szl_PL.UTF-8", "ta_IN", "ta_IN.UTF-8", "ta_LK", "ta_LK.UTF-8", "te_IN", "te_IN.UTF-8", "tg_TJ", "tg_TJ.KOI8-T", "tg_TJ.UTF-8", "the_NP", "the_NP.UTF-8", "th_TH", "th_TH.TIS-620", "th_TH.UTF-8", "ti_ER", "ti_ER.UTF-8", "ti_ET", "ti_ET.UTF-8", "tig_ER", "tig_ER.UTF-8", "tk_TM", "tk_TM.UTF-8", "tl_PH", "tl_PH.ISO-8859-1", "tl_PH.UTF-8", "tn_ZA", "tn_ZA.UTF-8", "to_TO", "to_TO.UTF-8", "tpi_PG", "tpi_PG.UTF-8", "tr_CY", "tr_CY.ISO-8859-9", "tr_CY.UTF-8", "tr_TR", "tr_TR.ISO-8859-9", "tr_TR.UTF-8", "ts_ZA", "ts_ZA.UTF-8", "tt_RU", "tt_RU.UTF-8", "ug_CN", "ug_CN.UTF-8", "uk_UA", "uk_UA.KOI8-U", "uk_UA.UTF-8", "unm_US", "unm_US.UTF-8", "ur_IN", "ur_IN.UTF-8", "ur_PK", "ur_PK.UTF-8", "uz_UZ", "uz_UZ.ISO-8859-1", "uz_UZ.UTF-8", "ve_ZA", "ve_ZA.UTF-8", "vi_VN", "vi_VN.UTF-8", "wa_BE", "wa_BE.ISO-8859-1", "wa_BE.UTF-8", "wae_CH", "wae_CH.UTF-8", "wal_ET", "wal_ET.UTF-8", "wo_SN", "wo_SN.UTF-8", "xh_ZA", "xh_ZA.ISO-8859-1", "xh_ZA.UTF-8", "yi_US", "yi_US.CP1255", "yi_US.UTF-8", "yo_NG", "yo_NG.UTF-8", "yue_HK", "yue_HK.UTF-8", "yuw_PG", "yuw_PG.UTF-8", "zh_CN", "zh_CN.GB18030", "zh_CN.GB2312", "zh_CN.GBK", "zh_CN.UTF-8", "zh_HK", "zh_HK.BIG5-HKSCS", "zh_HK.UTF-8", "zh_SG", "zh_SG.GB2312", "zh_SG.GBK", "zh_SG.UTF-8", "zh_TW", "zh_TW.BIG5", "zh_TW.EUC-TW", "zh_TW.UTF-8", "zu_ZA", "zu_ZA.ISO-8859-1", "zu_ZA.UTF-8"}; std::string ConsumeLocaleIdentifier(FuzzedDataProvider& fuzzed_data_provider) { @@ -56,7 +55,10 @@ void test_one_input(const std::vector<uint8_t>& buffer) const int64_t atoi64c_without_locale = atoi64(random_string.c_str()); const int64_t random_int64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); const std::string tostring_without_locale = ToString(random_int64); + // The variable `random_int32` is no longer used, but the harness still needs to + // consume the same data that it did previously to not invalidate existing seeds. const int32_t random_int32 = fuzzed_data_provider.ConsumeIntegral<int32_t>(); + (void)random_int32; const std::string strprintf_int_without_locale = strprintf("%d", random_int64); const double random_double = fuzzed_data_provider.ConsumeFloatingPoint<double>(); const std::string strprintf_double_without_locale = strprintf("%f", random_double); diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index 571364aaa6..a269378607 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/prevector.cpp b/src/test/fuzz/prevector.cpp new file mode 100644 index 0000000000..64920f4af5 --- /dev/null +++ b/src/test/fuzz/prevector.cpp @@ -0,0 +1,263 @@ +// Copyright (c) 2015-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> + +#include <prevector.h> +#include <vector> + +#include <reverse_iterator.h> +#include <serialize.h> +#include <streams.h> + +namespace { + +template<unsigned int N, typename T> +class prevector_tester { + typedef std::vector<T> realtype; + realtype real_vector; + realtype real_vector_alt; + + typedef prevector<N, T> pretype; + pretype pre_vector; + pretype pre_vector_alt; + + typedef typename pretype::size_type Size; + +public: + void test() const { + const pretype& const_pre_vector = pre_vector; + assert(real_vector.size() == pre_vector.size()); + assert(real_vector.empty() == pre_vector.empty()); + for (Size s = 0; s < real_vector.size(); s++) { + assert(real_vector[s] == pre_vector[s]); + assert(&(pre_vector[s]) == &(pre_vector.begin()[s])); + assert(&(pre_vector[s]) == &*(pre_vector.begin() + s)); + assert(&(pre_vector[s]) == &*((pre_vector.end() + s) - real_vector.size())); + } + // assert(realtype(pre_vector) == real_vector); + assert(pretype(real_vector.begin(), real_vector.end()) == pre_vector); + assert(pretype(pre_vector.begin(), pre_vector.end()) == pre_vector); + size_t pos = 0; + for (const T& v : pre_vector) { + assert(v == real_vector[pos]); + ++pos; + } + for (const T& v : reverse_iterate(pre_vector)) { + --pos; + assert(v == real_vector[pos]); + } + for (const T& v : const_pre_vector) { + assert(v == real_vector[pos]); + ++pos; + } + for (const T& v : reverse_iterate(const_pre_vector)) { + --pos; + assert(v == real_vector[pos]); + } + CDataStream ss1(SER_DISK, 0); + CDataStream ss2(SER_DISK, 0); + ss1 << real_vector; + ss2 << pre_vector; + assert(ss1.size() == ss2.size()); + for (Size s = 0; s < ss1.size(); s++) { + assert(ss1[s] == ss2[s]); + } + } + + void resize(Size s) { + real_vector.resize(s); + assert(real_vector.size() == s); + pre_vector.resize(s); + assert(pre_vector.size() == s); + } + + void reserve(Size s) { + real_vector.reserve(s); + assert(real_vector.capacity() >= s); + pre_vector.reserve(s); + assert(pre_vector.capacity() >= s); + } + + void insert(Size position, const T& value) { + real_vector.insert(real_vector.begin() + position, value); + pre_vector.insert(pre_vector.begin() + position, value); + } + + void insert(Size position, Size count, const T& value) { + real_vector.insert(real_vector.begin() + position, count, value); + pre_vector.insert(pre_vector.begin() + position, count, value); + } + + template<typename I> + void insert_range(Size position, I first, I last) { + real_vector.insert(real_vector.begin() + position, first, last); + pre_vector.insert(pre_vector.begin() + position, first, last); + } + + void erase(Size position) { + real_vector.erase(real_vector.begin() + position); + pre_vector.erase(pre_vector.begin() + position); + } + + void erase(Size first, Size last) { + real_vector.erase(real_vector.begin() + first, real_vector.begin() + last); + pre_vector.erase(pre_vector.begin() + first, pre_vector.begin() + last); + } + + void update(Size pos, const T& value) { + real_vector[pos] = value; + pre_vector[pos] = value; + } + + void push_back(const T& value) { + real_vector.push_back(value); + pre_vector.push_back(value); + } + + void pop_back() { + real_vector.pop_back(); + pre_vector.pop_back(); + } + + void clear() { + real_vector.clear(); + pre_vector.clear(); + } + + void assign(Size n, const T& value) { + real_vector.assign(n, value); + pre_vector.assign(n, value); + } + + Size size() const { + return real_vector.size(); + } + + Size capacity() const { + return pre_vector.capacity(); + } + + void shrink_to_fit() { + pre_vector.shrink_to_fit(); + } + + void swap() { + real_vector.swap(real_vector_alt); + pre_vector.swap(pre_vector_alt); + } + + void move() { + real_vector = std::move(real_vector_alt); + real_vector_alt.clear(); + pre_vector = std::move(pre_vector_alt); + pre_vector_alt.clear(); + } + + void copy() { + real_vector = real_vector_alt; + pre_vector = pre_vector_alt; + } + + void resize_uninitialized(realtype values) { + size_t r = values.size(); + size_t s = real_vector.size() / 2; + if (real_vector.capacity() < s + r) { + real_vector.reserve(s + r); + } + real_vector.resize(s); + pre_vector.resize_uninitialized(s); + for (auto v : values) { + real_vector.push_back(v); + } + auto p = pre_vector.size(); + pre_vector.resize_uninitialized(p + r); + for (auto v : values) { + pre_vector[p] = v; + ++p; + } + } +}; + +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider prov(buffer.data(), buffer.size()); + prevector_tester<8, int> test; + + while (prov.remaining_bytes()) { + switch (prov.ConsumeIntegralInRange<int>(0, 13 + 3 * (test.size() > 0))) { + case 0: + test.insert(prov.ConsumeIntegralInRange<size_t>(0, test.size()), prov.ConsumeIntegral<int>()); + break; + case 1: + test.resize(std::max(0, std::min(30, (int)test.size() + prov.ConsumeIntegralInRange<int>(0, 4) - 2))); + break; + case 2: + test.insert(prov.ConsumeIntegralInRange<size_t>(0, test.size()), 1 + prov.ConsumeBool(), prov.ConsumeIntegral<int>()); + break; + case 3: { + int del = prov.ConsumeIntegralInRange<int>(0, test.size()); + int beg = prov.ConsumeIntegralInRange<int>(0, test.size() - del); + test.erase(beg, beg + del); + break; + } + case 4: + test.push_back(prov.ConsumeIntegral<int>()); + break; + case 5: { + int values[4]; + int num = 1 + prov.ConsumeIntegralInRange<int>(0, 3); + for (int k = 0; k < num; ++k) { + values[k] = prov.ConsumeIntegral<int>(); + } + test.insert_range(prov.ConsumeIntegralInRange<size_t>(0, test.size()), values, values + num); + break; + } + case 6: { + int num = 1 + prov.ConsumeIntegralInRange<int>(0, 15); + std::vector<int> values(num); + for (auto& v : values) { + v = prov.ConsumeIntegral<int>(); + } + test.resize_uninitialized(values); + break; + } + case 7: + test.reserve(prov.ConsumeIntegralInRange<size_t>(0, 32767)); + break; + case 8: + test.shrink_to_fit(); + break; + case 9: + test.clear(); + break; + case 10: + test.assign(prov.ConsumeIntegralInRange<size_t>(0, 32767), prov.ConsumeIntegral<int>()); + break; + case 11: + test.swap(); + break; + case 12: + test.copy(); + break; + case 13: + test.move(); + break; + case 14: + test.update(prov.ConsumeIntegralInRange<size_t>(0, test.size() - 1), prov.ConsumeIntegral<int>()); + break; + case 15: + test.erase(prov.ConsumeIntegralInRange<size_t>(0, test.size() - 1)); + break; + case 16: + test.pop_back(); + break; + } + } + + test.test(); +} diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index dc49dd499a..0ed3721636 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -32,7 +32,7 @@ #include <string> #include <vector> -bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); +bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); namespace { @@ -57,12 +57,17 @@ const std::map<std::string, std::set<std::string>> EXPECTED_DESERIALIZATION_EXCE {"Unknown transaction optional data: iostream error", {"block", "blocktxn", "cmpctblock", "tx"}}, }; -const RegTestingSetup* g_setup; +const TestingSetup* g_setup; } // namespace void initialize() { - static RegTestingSetup setup{}; + static TestingSetup setup{ + CBaseChainParams::REGTEST, + { + "-nodebuglogfile", + }, + }; g_setup = &setup; for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp new file mode 100644 index 0000000000..bcbf65bdca --- /dev/null +++ b/src/test/fuzz/process_messages.cpp @@ -0,0 +1,80 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/consensus.h> +#include <net.h> +#include <net_processing.h> +#include <protocol.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/mining.h> +#include <test/util/net.h> +#include <test/util/setup_common.h> +#include <util/memory.h> +#include <validation.h> +#include <validationinterface.h> + +const TestingSetup* g_setup; + +void initialize() +{ + static TestingSetup setup{ + CBaseChainParams::REGTEST, + { + "-nodebuglogfile", + }, + }; + g_setup = &setup; + + for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { + MineBlock(g_setup->m_node, CScript() << OP_TRUE); + } + SyncWithValidationInterfaceQueue(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); + std::vector<CNode*> peers; + + const auto num_peers_to_add = fuzzed_data_provider.ConsumeIntegralInRange(1, 3); + for (int i = 0; i < num_peers_to_add; ++i) { + const ServiceFlags service_flags = ServiceFlags(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + const bool inbound{fuzzed_data_provider.ConsumeBool()}; + const bool block_relay_only{fuzzed_data_provider.ConsumeBool()}; + peers.push_back(MakeUnique<CNode>(i, service_flags, 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, inbound, block_relay_only).release()); + CNode& p2p_node = *peers.back(); + + p2p_node.fSuccessfullyConnected = true; + p2p_node.fPauseSend = false; + p2p_node.nVersion = PROTOCOL_VERSION; + p2p_node.SetSendVersion(PROTOCOL_VERSION); + g_setup->m_node.peer_logic->InitializeNode(&p2p_node); + + connman.AddTestNode(p2p_node); + } + + while (fuzzed_data_provider.ConsumeBool()) { + const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; + + CSerializedNetMsg net_msg; + net_msg.command = random_message_type; + net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + + CNode& random_node = *peers.at(fuzzed_data_provider.ConsumeIntegralInRange<int>(0, peers.size() - 1)); + + (void)connman.ReceiveMsgFrom(random_node, net_msg); + random_node.fPauseSend = false; + + try { + connman.ProcessMessagesOnce(random_node); + } catch (const std::ios_base::failure&) { + } + } + connman.ClearTestNodes(); + SyncWithValidationInterfaceQueue(); +} diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index ca3e0b8586..64328fb66e 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 80e2f234d7..de82122dd6 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 3d8ece7c61..ffc65eedc0 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/signature_checker.cpp b/src/test/fuzz/signature_checker.cpp index 312db27adc..4a8c7a63af 100644 --- a/src/test/fuzz/signature_checker.cpp +++ b/src/test/fuzz/signature_checker.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index d8e84f1a0f..d6deb7fc3d 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index b70ea6d90e..9c7b0d47a2 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -120,4 +120,15 @@ NODISCARD bool MultiplicationOverflow(const T i, const T j) noexcept } } +template <class T> +NODISCARD bool AdditionOverflow(const T i, const T j) noexcept +{ + static_assert(std::is_integral<T>::value, "Integral required."); + if (std::numeric_limits<T>::is_signed) { + return (i > 0 && j > std::numeric_limits<T>::max() - i) || + (i < 0 && j < std::numeric_limits<T>::min() - i); + } + return std::numeric_limits<T>::max() - i < j; +} + #endif // BITCOIN_TEST_FUZZ_UTIL_H diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 10fb05ca8a..512e48f8e5 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <test/util/setup_common.h> #include <util/strencodings.h> #include <util/system.h> -#include <test/util/setup_common.h> #include <string> #include <utility> diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index b864e6e599..87f6470afa 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2013-2019 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <clientversion.h> #include <crypto/siphash.h> #include <hash.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/interfaces_tests.cpp b/src/test/interfaces_tests.cpp new file mode 100644 index 0000000000..b0d4de89f3 --- /dev/null +++ b/src/test/interfaces_tests.cpp @@ -0,0 +1,163 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <consensus/validation.h> +#include <interfaces/chain.h> +#include <script/standard.h> +#include <test/util/setup_common.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> + +using interfaces::FoundBlock; + +BOOST_FIXTURE_TEST_SUITE(interfaces_tests, TestChain100Setup) + +BOOST_AUTO_TEST_CASE(findBlock) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + + uint256 hash; + BOOST_CHECK(chain->findBlock(active[10]->GetBlockHash(), FoundBlock().hash(hash))); + BOOST_CHECK_EQUAL(hash, active[10]->GetBlockHash()); + + int height = -1; + BOOST_CHECK(chain->findBlock(active[20]->GetBlockHash(), FoundBlock().height(height))); + BOOST_CHECK_EQUAL(height, active[20]->nHeight); + + CBlock data; + BOOST_CHECK(chain->findBlock(active[30]->GetBlockHash(), FoundBlock().data(data))); + BOOST_CHECK_EQUAL(data.GetHash(), active[30]->GetBlockHash()); + + int64_t time = -1; + BOOST_CHECK(chain->findBlock(active[40]->GetBlockHash(), FoundBlock().time(time))); + BOOST_CHECK_EQUAL(time, active[40]->GetBlockTime()); + + int64_t max_time = -1; + BOOST_CHECK(chain->findBlock(active[50]->GetBlockHash(), FoundBlock().maxTime(max_time))); + BOOST_CHECK_EQUAL(max_time, active[50]->GetBlockTimeMax()); + + int64_t mtp_time = -1; + BOOST_CHECK(chain->findBlock(active[60]->GetBlockHash(), FoundBlock().mtpTime(mtp_time))); + BOOST_CHECK_EQUAL(mtp_time, active[60]->GetMedianTimePast()); + + BOOST_CHECK(!chain->findBlock({}, FoundBlock())); +} + +BOOST_AUTO_TEST_CASE(findFirstBlockWithTimeAndHeight) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + uint256 hash; + int height; + BOOST_CHECK(chain->findFirstBlockWithTimeAndHeight(/* min_time= */ 0, /* min_height= */ 5, FoundBlock().hash(hash).height(height))); + BOOST_CHECK_EQUAL(hash, active[5]->GetBlockHash()); + BOOST_CHECK_EQUAL(height, 5); + BOOST_CHECK(!chain->findFirstBlockWithTimeAndHeight(/* min_time= */ active.Tip()->GetBlockTimeMax() + 1, /* min_height= */ 0)); +} + +BOOST_AUTO_TEST_CASE(findNextBlock) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + bool reorg; + uint256 hash; + BOOST_CHECK(chain->findNextBlock(active[20]->GetBlockHash(), 20, FoundBlock().hash(hash), &reorg)); + BOOST_CHECK_EQUAL(hash, active[21]->GetBlockHash()); + BOOST_CHECK_EQUAL(reorg, false); + BOOST_CHECK(!chain->findNextBlock(uint256(), 20, {}, &reorg)); + BOOST_CHECK_EQUAL(reorg, true); + BOOST_CHECK(!chain->findNextBlock(active.Tip()->GetBlockHash(), active.Height(), {}, &reorg)); + BOOST_CHECK_EQUAL(reorg, false); +} + +BOOST_AUTO_TEST_CASE(findAncestorByHeight) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + uint256 hash; + BOOST_CHECK(chain->findAncestorByHeight(active[20]->GetBlockHash(), 10, FoundBlock().hash(hash))); + BOOST_CHECK_EQUAL(hash, active[10]->GetBlockHash()); + BOOST_CHECK(!chain->findAncestorByHeight(active[10]->GetBlockHash(), 20)); +} + +BOOST_AUTO_TEST_CASE(findAncestorByHash) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + int height = -1; + BOOST_CHECK(chain->findAncestorByHash(active[20]->GetBlockHash(), active[10]->GetBlockHash(), FoundBlock().height(height))); + BOOST_CHECK_EQUAL(height, 10); + BOOST_CHECK(!chain->findAncestorByHash(active[10]->GetBlockHash(), active[20]->GetBlockHash())); +} + +BOOST_AUTO_TEST_CASE(findCommonAncestor) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + auto* orig_tip = active.Tip(); + for (int i = 0; i < 10; ++i) { + BlockValidationState state; + ChainstateActive().InvalidateBlock(state, Params(), active.Tip()); + } + BOOST_CHECK_EQUAL(active.Height(), orig_tip->nHeight - 10); + coinbaseKey.MakeNewKey(true); + for (int i = 0; i < 20; ++i) { + CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + } + BOOST_CHECK_EQUAL(active.Height(), orig_tip->nHeight + 10); + uint256 fork_hash; + int fork_height; + int orig_height; + BOOST_CHECK(chain->findCommonAncestor(orig_tip->GetBlockHash(), active.Tip()->GetBlockHash(), FoundBlock().height(fork_height).hash(fork_hash), FoundBlock().height(orig_height))); + BOOST_CHECK_EQUAL(orig_height, orig_tip->nHeight); + BOOST_CHECK_EQUAL(fork_height, orig_tip->nHeight - 10); + BOOST_CHECK_EQUAL(fork_hash, active[fork_height]->GetBlockHash()); + + uint256 active_hash, orig_hash; + BOOST_CHECK(!chain->findCommonAncestor(active.Tip()->GetBlockHash(), {}, {}, FoundBlock().hash(active_hash), {})); + BOOST_CHECK(!chain->findCommonAncestor({}, orig_tip->GetBlockHash(), {}, {}, FoundBlock().hash(orig_hash))); + BOOST_CHECK_EQUAL(active_hash, active.Tip()->GetBlockHash()); + BOOST_CHECK_EQUAL(orig_hash, orig_tip->GetBlockHash()); +} + +BOOST_AUTO_TEST_CASE(hasBlocks) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + + // Test ranges + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + active[5]->nStatus &= ~BLOCK_HAVE_DATA; + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + active[95]->nStatus &= ~BLOCK_HAVE_DATA; + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + active[50]->nStatus &= ~BLOCK_HAVE_DATA; + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + + // Test edge cases + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 6, 49)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 5, 49)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 6, 50)); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index b52513f4af..d465ee6759 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <key.h> #include <key_io.h> #include <script/script.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 034b7938f9..cf2bd03698 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> #include <key_io.h> +#include <test/util/setup_common.h> #include <uint256.h> -#include <util/system.h> #include <util/strencodings.h> #include <util/string.h> -#include <test/util/setup_common.h> +#include <util/system.h> #include <string> #include <vector> diff --git a/src/test/main.cpp b/src/test/main.cpp index f32243d1d3..5885564074 100644 --- a/src/test/main.cpp +++ b/src/test/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/merkleblock_tests.cpp b/src/test/merkleblock_tests.cpp index 9f8c4ba5c5..98b27994a6 100644 --- a/src/test/merkleblock_tests.cpp +++ b/src/test/merkleblock_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <merkleblock.h> -#include <uint256.h> #include <test/util/setup_common.h> +#include <uint256.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index 97a918da45..dd2890c134 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> #include <policy/policy.h> +#include <script/interpreter.h> #include <script/script.h> #include <script/script_error.h> -#include <script/interpreter.h> #include <script/sign.h> #include <script/signingprovider.h> +#include <test/util/setup_common.h> #include <tinyformat.h> #include <uint256.h> -#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 9b5a86fef2..84bf593497 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -1,23 +1,24 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrdb.h> #include <addrman.h> +#include <chainparams.h> #include <clientversion.h> -#include <test/util/setup_common.h> -#include <string> -#include <boost/test/unit_test.hpp> -#include <serialize.h> -#include <streams.h> #include <net.h> #include <netbase.h> -#include <chainparams.h> +#include <serialize.h> +#include <streams.h> +#include <test/util/setup_common.h> #include <util/memory.h> -#include <util/system.h> #include <util/string.h> +#include <util/system.h> + +#include <boost/test/unit_test.hpp> #include <memory> +#include <string> class CAddrManSerializationMock : public CAddrMan { diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 9730b40580..ec6a290334 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <netbase.h> #include <net_permissions.h> +#include <netbase.h> #include <test/util/setup_common.h> #include <util/strencodings.h> diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index bf58bd63b9..a9d661438c 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <arith_uint256.h> #include <consensus/merkle.h> #include <merkleblock.h> #include <serialize.h> #include <streams.h> +#include <test/util/setup_common.h> #include <uint256.h> -#include <arith_uint256.h> #include <version.h> -#include <test/util/setup_common.h> #include <vector> diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 025e2b78ca..06877898a4 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <policy/policy.h> #include <policy/fees.h> +#include <policy/policy.h> #include <txmempool.h> #include <uint256.h> #include <util/time.h> diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 9782b78f2c..12c5848eaf 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <vector> #include <prevector.h> +#include <vector> #include <reverse_iterator.h> #include <serialize.h> diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index e0df41a971..ca3b92f2e1 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <boost/test/unit_test.hpp> -#include <random> #include <algorithm> +#include <random> BOOST_FIXTURE_TEST_SUITE(random_tests, BasicTestingSetup) diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp index 4e51b8c02a..a42608a66d 100644 --- a/src/test/reverselock_tests.cpp +++ b/src/test/reverselock_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 84a3980b19..d9c66f1c19 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <rpc/server.h> #include <rpc/client.h> +#include <rpc/server.h> #include <rpc/util.h> #include <core_io.h> diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 801cf8e5d1..1395a7f38c 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,15 +6,17 @@ #include <scheduler.h> #include <util/time.h> -#include <boost/thread.hpp> #include <boost/test/unit_test.hpp> +#include <boost/thread.hpp> + +#include <mutex> BOOST_AUTO_TEST_SUITE(scheduler_tests) -static void microTask(CScheduler& s, boost::mutex& mutex, int& counter, int delta, std::chrono::system_clock::time_point rescheduleTime) +static void microTask(CScheduler& s, std::mutex& mutex, int& counter, int delta, std::chrono::system_clock::time_point rescheduleTime) { { - boost::unique_lock<boost::mutex> lock(mutex); + std::lock_guard<std::mutex> lock(mutex); counter += delta; } std::chrono::system_clock::time_point noTime = std::chrono::system_clock::time_point::min(); @@ -38,7 +40,7 @@ BOOST_AUTO_TEST_CASE(manythreads) // counters should sum to the number of initial tasks performed. CScheduler microTasks; - boost::mutex counterMutex[10]; + std::mutex counterMutex[10]; int counter[10] = { 0 }; FastRandomContext rng{/* fDeterministic */ true}; auto zeroToNine = [](FastRandomContext& rc) -> int { return rc.randrange(10); }; // [0, 9] diff --git a/src/test/script_p2sh_tests.cpp b/src/test/script_p2sh_tests.cpp index 8c1e843b0b..f6824a4e5e 100644 --- a/src/test/script_p2sh_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/tx_verify.h> #include <key.h> -#include <validation.h> #include <policy/policy.h> +#include <policy/settings.h> #include <script/script.h> #include <script/script_error.h> -#include <policy/settings.h> #include <script/sign.h> #include <script/signingprovider.h> #include <test/util/setup_common.h> +#include <validation.h> #include <vector> diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 26015ca4c2..4c2d35c502 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,16 +6,16 @@ #include <core_io.h> #include <key.h> +#include <rpc/util.h> #include <script/script.h> #include <script/script_error.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <util/system.h> -#include <util/strencodings.h> -#include <test/util/transaction_utils.h> -#include <test/util/setup_common.h> -#include <rpc/util.h> #include <streams.h> +#include <test/util/setup_common.h> +#include <test/util/transaction_utils.h> +#include <util/strencodings.h> +#include <util/system.h> #if defined(HAVE_CONSENSUS_LIB) #include <script/bitcoinconsensus.h> diff --git a/src/test/scriptnum10.h b/src/test/scriptnum10.h index 9f928827cb..352797f18d 100644 --- a/src/test/scriptnum10.h +++ b/src/test/scriptnum10.h @@ -1,17 +1,17 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TEST_SCRIPTNUM10_H #define BITCOIN_TEST_SCRIPTNUM10_H +#include <assert.h> #include <limits> #include <stdexcept> #include <stdint.h> #include <string> #include <vector> -#include <assert.h> class scriptnum10_error : public std::runtime_error { diff --git a/src/test/scriptnum_tests.cpp b/src/test/scriptnum_tests.cpp index 40a6f69668..281018be9f 100644 --- a/src/test/scriptnum_tests.cpp +++ b/src/test/scriptnum_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/scriptnum10.h> #include <script/script.h> +#include <test/scriptnum10.h> #include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index ea600499ca..9a6c721ab8 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <hash.h> #include <serialize.h> #include <streams.h> -#include <hash.h> #include <test/util/setup_common.h> #include <util/strencodings.h> diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 10b161aa80..fcd831ccda 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index bcc4a46873..5ca136ea6e 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -1,18 +1,18 @@ -// Copyright (c) 2013-2019 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/tx_check.h> #include <consensus/validation.h> -#include <test/data/sighash.json.h> #include <hash.h> #include <script/interpreter.h> #include <script/script.h> #include <serialize.h> #include <streams.h> +#include <test/data/sighash.json.h> #include <test/util/setup_common.h> -#include <util/system.h> #include <util/strencodings.h> +#include <util/system.h> #include <version.h> #include <iostream> diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 6462fcefe3..6e36bce7a1 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/consensus.h> #include <consensus/tx_verify.h> -#include <pubkey.h> #include <key.h> +#include <pubkey.h> #include <script/script.h> #include <script/standard.h> -#include <uint256.h> #include <test/util/setup_common.h> +#include <uint256.h> #include <vector> diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp index 29b43e9bec..8c880babd1 100644 --- a/src/test/timedata_tests.cpp +++ b/src/test/timedata_tests.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include <netaddress.h> #include <noui.h> -#include <util/string.h> #include <test/util/logging.h> #include <test/util/setup_common.h> #include <timedata.h> +#include <util/string.h> #include <warnings.h> #include <string> diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 96520079d7..9d0ae56c3f 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,23 +6,23 @@ #include <test/data/tx_valid.json.h> #include <test/util/setup_common.h> -#include <clientversion.h> #include <checkqueue.h> +#include <clientversion.h> #include <consensus/tx_check.h> #include <consensus/validation.h> #include <core_io.h> #include <key.h> -#include <validation.h> #include <policy/policy.h> #include <policy/settings.h> #include <script/script.h> +#include <script/script_error.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/script_error.h> #include <script/standard.h> #include <streams.h> -#include <util/strencodings.h> #include <test/util/transaction_utils.h> +#include <util/strencodings.h> +#include <validation.h> #include <map> #include <string> diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 3550a02316..5fc172ee86 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -70,12 +70,8 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) // shutdown sequence (c.f. Shutdown() in init.cpp) txindex.Stop(); - // txindex job may be scheduled, so stop scheduler before destructing - m_node.scheduler->stop(); - threadGroup.interrupt_all(); - threadGroup.join_all(); - - // Rest of shutdown sequence and destructors happen in ~TestingSetup() + // Let scheduler events finish running to avoid accessing any memory related to txindex after it is destructed + SyncWithValidationInterfaceQueue(); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index cace75f093..c3d7af8323 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <validation.h> #include <consensus/validation.h> #include <primitives/transaction.h> #include <script/script.h> #include <test/util/setup_common.h> +#include <validation.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 7842594b80..cdef7dcc3c 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -4,12 +4,12 @@ #include <consensus/validation.h> #include <key.h> -#include <validation.h> -#include <txmempool.h> -#include <script/standard.h> #include <script/sign.h> #include <script/signingprovider.h> +#include <script/standard.h> #include <test/util/setup_common.h> +#include <txmempool.h> +#include <validation.h> #include <boost/test/unit_test.hpp> @@ -108,7 +108,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // any script flag that is implemented as an upgraded NOP code. static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - PrecomputedTransactionData txdata(tx); + PrecomputedTransactionData txdata; // If we add many more flags, this loop can get too expensive, but we can // rewrite in the future to randomly pick a set of flags to evaluate. for (uint32_t test_flags=0; test_flags < (1U << 16); test_flags += 1) { @@ -200,7 +200,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) LOCK(cs_main); TxValidationState state; - PrecomputedTransactionData ptd_spend_tx(spend_tx); + PrecomputedTransactionData ptd_spend_tx; BOOST_CHECK(!CheckInputScripts(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); @@ -269,7 +269,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 100; TxValidationState state; - PrecomputedTransactionData txdata(invalid_with_cltv_tx); + PrecomputedTransactionData txdata; BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_cltv_tx), state, ::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); } @@ -297,7 +297,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 100; TxValidationState state; - PrecomputedTransactionData txdata(invalid_with_csv_tx); + PrecomputedTransactionData txdata; BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_csv_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); } @@ -358,7 +358,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) tx.vin[1].scriptWitness.SetNull(); TxValidationState state; - PrecomputedTransactionData txdata(tx); + PrecomputedTransactionData txdata; // This transaction is now invalid under segwit, because of the second input. BOOST_CHECK(!CheckInputScripts(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 7293ecd325..c0ae2f8cf2 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -1,16 +1,17 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <arith_uint256.h> #include <streams.h> +#include <test/util/setup_common.h> #include <uint256.h> #include <version.h> -#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> -#include <sstream> + #include <iomanip> +#include <sstream> #include <string> BOOST_FIXTURE_TEST_SUITE(uint256_tests, BasicTestingSetup) diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp new file mode 100644 index 0000000000..09f2f1807f --- /dev/null +++ b/src/test/util/net.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/net.h> + +#include <chainparams.h> +#include <net.h> + +void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, const char* pch, unsigned int nBytes, bool& complete) const +{ + assert(node.ReceiveMsgBytes(pch, nBytes, complete)); + if (complete) { + size_t nSizeAdded = 0; + auto it(node.vRecvMsg.begin()); + for (; it != node.vRecvMsg.end(); ++it) { + // vRecvMsg contains only completed CNetMessage + // the single possible partially deserialized message are held by TransportDeserializer + nSizeAdded += it->m_raw_message_size; + } + { + LOCK(node.cs_vProcessMsg); + node.vProcessMsg.splice(node.vProcessMsg.end(), node.vRecvMsg, node.vRecvMsg.begin(), it); + node.nProcessQueueSize += nSizeAdded; + node.fPauseRecv = node.nProcessQueueSize > nReceiveFloodSize; + } + } +} + +bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) const +{ + std::vector<unsigned char> ser_msg_header; + node.m_serializer->prepareForTransport(ser_msg, ser_msg_header); + + bool complete; + NodeReceiveMsgBytes(node, (const char*)ser_msg_header.data(), ser_msg_header.size(), complete); + NodeReceiveMsgBytes(node, (const char*)ser_msg.data.data(), ser_msg.data.size(), complete); + return complete; +} diff --git a/src/test/util/net.h b/src/test/util/net.h new file mode 100644 index 0000000000..ca8cb7fad5 --- /dev/null +++ b/src/test/util/net.h @@ -0,0 +1,33 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_NET_H +#define BITCOIN_TEST_UTIL_NET_H + +#include <net.h> + +struct ConnmanTestMsg : public CConnman { + using CConnman::CConnman; + void AddTestNode(CNode& node) + { + LOCK(cs_vNodes); + vNodes.push_back(&node); + } + void ClearTestNodes() + { + LOCK(cs_vNodes); + for (CNode* node : vNodes) { + delete node; + } + vNodes.clear(); + } + + void ProcessMessagesOnce(CNode& node) { m_msgproc->ProcessMessages(&node, flagInterruptMsgProc); } + + void NodeReceiveMsgBytes(CNode& node, const char* pch, unsigned int nBytes, bool& complete) const; + + bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) const; +}; + +#endif // BITCOIN_TEST_UTIL_NET_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index d684b97787..bf0afc4171 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -27,12 +27,15 @@ #include <util/string.h> #include <util/time.h> #include <util/translation.h> +#include <util/url.h> +#include <util/vector.h> #include <validation.h> #include <validationinterface.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = nullptr; FastRandomContext g_insecure_rand_ctx; /** Random context to get unique temp data dirs. Separate from g_insecure_rand_ctx, which can be seeded from a const env var */ @@ -63,17 +66,34 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) return os; } -BasicTestingSetup::BasicTestingSetup(const std::string& chainName) +BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) : m_path_root{fs::temp_directory_path() / "test_common_" PACKAGE_NAME / g_insecure_rand_ctx_temp_path.rand256().ToString()} { + const std::vector<const char*> arguments = Cat( + { + "dummy", + "-printtoconsole=0", + "-logtimemicros", + "-debug", + "-debugexclude=libevent", + "-debugexclude=leveldb", + }, + extra_args); fs::create_directories(m_path_root); gArgs.ForceSetArg("-datadir", m_path_root.string()); ClearDatadirCache(); + { + SetupServerArgs(m_node); + std::string error; + const bool success{m_node.args->ParseParameters(arguments.size(), arguments.data(), error)}; + assert(success); + assert(error.empty()); + } SelectParams(chainName); SeedInsecureRand(); - gArgs.ForceSetArg("-printtoconsole", "0"); if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN); InitLogging(); + AppInitParameterInteraction(); LogInstance().StartLogging(); SHA256AutoDetect(); ECC_Start(); @@ -93,10 +113,12 @@ BasicTestingSetup::~BasicTestingSetup() { LogInstance().DisconnectTestLogger(); fs::remove_all(m_path_root); + gArgs.ClearArgs(); ECC_Stop(); } -TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) +TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) + : BasicTestingSetup(chainName, extra_args) { const CChainParams& chainparams = Params(); // Ideally we'd move all the RPC tests to the functional testing framework @@ -112,7 +134,8 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha GetMainSignals().RegisterBackgroundSignalScheduler(*g_rpc_node->scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - g_chainstate = MakeUnique<CChainState>(); + + g_chainman.InitializeChainstate(); ::ChainstateActive().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!::ChainstateActive().CanFlushToDisk()); @@ -139,6 +162,11 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. m_node.peer_logic = MakeUnique<PeerLogicValidation>(m_node.connman.get(), m_node.banman.get(), *m_node.scheduler, *m_node.mempool); + { + CConnman::Options options; + options.m_msgproc = m_node.peer_logic.get(); + m_node.connman->Init(options); + } } TestingSetup::~TestingSetup() @@ -151,10 +179,11 @@ TestingSetup::~TestingSetup() g_rpc_node = nullptr; m_node.connman.reset(); m_node.banman.reset(); + m_node.args = nullptr; m_node.mempool = nullptr; m_node.scheduler.reset(); UnloadBlockIndex(); - g_chainstate.reset(); + g_chainman.Reset(); pblocktree.reset(); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 0930309c3a..2477f9ad06 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -73,9 +73,11 @@ static constexpr CAmount CENT{1000000}; */ struct BasicTestingSetup { ECCVerifyHandle globalVerifyHandle; + NodeContext m_node; - explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~BasicTestingSetup(); + private: const fs::path m_path_root; }; @@ -84,10 +86,9 @@ private: * Included are coins database, script check threads setup. */ struct TestingSetup : public BasicTestingSetup { - NodeContext m_node; boost::thread_group threadGroup; - explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~TestingSetup(); }; diff --git a/src/test/util/transaction_utils.cpp b/src/test/util/transaction_utils.cpp index 999b803a8d..7e5bb30a2c 100644 --- a/src/test/util/transaction_utils.cpp +++ b/src/test/util/transaction_utils.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/util/transaction_utils.h> #include <coins.h> #include <script/signingprovider.h> +#include <test/util/transaction_utils.h> CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int nValue) { diff --git a/src/test/util/transaction_utils.h b/src/test/util/transaction_utils.h index f843928a5f..1beddd334b 100644 --- a/src/test/util/transaction_utils.h +++ b/src/test/util/transaction_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 73b37f909f..45b7fd4932 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -6,7 +6,7 @@ #include <clientversion.h> #include <hash.h> // For Hash() -#include <key.h> // For CKey +#include <key.h> // For CKey #include <optional.h> #include <sync.h> #include <test/util/setup_common.h> @@ -14,10 +14,10 @@ #include <uint256.h> #include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> +#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> -#include <util/spanparsing.h> #include <util/vector.h> #include <array> diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index cee4e0ce3c..f226caf717 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <test/util/setup_common.h> #include <util/string.h> #include <util/threadnames.h> -#include <test/util/setup_common.h> +#include <mutex> +#include <set> #include <thread> #include <vector> -#include <set> -#include <mutex> #if defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index afb3db36a2..b8af869b8e 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp new file mode 100644 index 0000000000..0d149285ad --- /dev/null +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -0,0 +1,107 @@ +// Copyright (c) 2019-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include <chainparams.h> +#include <consensus/validation.h> +#include <random.h> +#include <sync.h> +#include <test/util/setup_common.h> +#include <uint256.h> +#include <validation.h> +#include <validationinterface.h> + +#include <vector> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, TestingSetup) + +//! Basic tests for ChainstateManager. +//! +//! First create a legacy (IBD) chainstate, then create a snapshot chainstate. +BOOST_AUTO_TEST_CASE(chainstatemanager) +{ + ChainstateManager manager; + std::vector<CChainState*> chainstates; + const CChainParams& chainparams = Params(); + + // Create a legacy (IBD) chainstate. + // + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c1 = manager.InitializeChainstate(); + LEAVE_CRITICAL_SECTION(cs_main); + chainstates.push_back(&c1); + c1.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + WITH_LOCK(::cs_main, c1.InitCoinsCache()); + + BOOST_CHECK(!manager.IsSnapshotActive()); + BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(!manager.IsBackgroundIBD(&c1)); + auto all = manager.GetAll(); + BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); + + auto& active_chain = manager.ActiveChain(); + BOOST_CHECK_EQUAL(&active_chain, &c1.m_chain); + + BOOST_CHECK_EQUAL(manager.ActiveHeight(), -1); + + auto active_tip = manager.ActiveTip(); + auto exp_tip = c1.m_chain.Tip(); + BOOST_CHECK_EQUAL(active_tip, exp_tip); + + auto& validated_cs = manager.ValidatedChainstate(); + BOOST_CHECK_EQUAL(&validated_cs, &c1); + + // Create a snapshot-based chainstate. + // + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c2 = manager.InitializeChainstate(GetRandHash()); + LEAVE_CRITICAL_SECTION(cs_main); + chainstates.push_back(&c2); + c2.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + WITH_LOCK(::cs_main, c2.InitCoinsCache()); + // Unlike c1, which doesn't have any blocks. Gets us different tip, height. + c2.LoadGenesisBlock(chainparams); + BlockValidationState _; + BOOST_CHECK(c2.ActivateBestChain(_, chainparams, nullptr)); + + BOOST_CHECK(manager.IsSnapshotActive()); + BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(manager.IsBackgroundIBD(&c1)); + BOOST_CHECK(!manager.IsBackgroundIBD(&c2)); + auto all2 = manager.GetAll(); + BOOST_CHECK_EQUAL_COLLECTIONS(all2.begin(), all2.end(), chainstates.begin(), chainstates.end()); + + auto& active_chain2 = manager.ActiveChain(); + BOOST_CHECK_EQUAL(&active_chain2, &c2.m_chain); + + BOOST_CHECK_EQUAL(manager.ActiveHeight(), 0); + + auto active_tip2 = manager.ActiveTip(); + auto exp_tip2 = c2.m_chain.Tip(); + BOOST_CHECK_EQUAL(active_tip2, exp_tip2); + + // Ensure that these pointers actually correspond to different + // CCoinsViewCache instances. + BOOST_CHECK(exp_tip != exp_tip2); + + auto& validated_cs2 = manager.ValidatedChainstate(); + BOOST_CHECK_EQUAL(&validated_cs2, &c1); + + auto& validated_chain = manager.ValidatedChain(); + BOOST_CHECK_EQUAL(&validated_chain, &c1.m_chain); + + auto validated_tip = manager.ValidatedTip(); + exp_tip = c1.m_chain.Tip(); + BOOST_CHECK_EQUAL(validated_tip, exp_tip); + + // Let scheduler events finish running to avoid accessing memory that is going to be unloaded + SyncWithValidationInterfaceQueue(); + + WITH_LOCK(::cs_main, manager.Unload()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index c24164528f..388a2dbd13 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // diff --git a/src/test/validationinterface_tests.cpp b/src/test/validationinterface_tests.cpp new file mode 100644 index 0000000000..208be92852 --- /dev/null +++ b/src/test/validationinterface_tests.cpp @@ -0,0 +1,60 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <boost/test/unit_test.hpp> +#include <consensus/validation.h> +#include <primitives/block.h> +#include <scheduler.h> +#include <test/util/setup_common.h> +#include <util/check.h> +#include <validationinterface.h> + +BOOST_FIXTURE_TEST_SUITE(validationinterface_tests, TestingSetup) + +class TestInterface : public CValidationInterface +{ +public: + TestInterface(std::function<void()> on_call = nullptr, std::function<void()> on_destroy = nullptr) + : m_on_call(std::move(on_call)), m_on_destroy(std::move(on_destroy)) + { + } + virtual ~TestInterface() + { + if (m_on_destroy) m_on_destroy(); + } + void BlockChecked(const CBlock& block, const BlockValidationState& state) override + { + if (m_on_call) m_on_call(); + } + static void Call() + { + CBlock block; + BlockValidationState state; + GetMainSignals().BlockChecked(block, state); + } + std::function<void()> m_on_call; + std::function<void()> m_on_destroy; +}; + +// Regression test to ensure UnregisterAllValidationInterfaces calls don't +// destroy a validation interface while it is being called. Bug: +// https://github.com/bitcoin/bitcoin/pull/18551 +BOOST_AUTO_TEST_CASE(unregister_all_during_call) +{ + bool destroyed = false; + RegisterSharedValidationInterface(std::make_shared<TestInterface>( + [&] { + // First call should decrements reference count 2 -> 1 + UnregisterAllValidationInterfaces(); + BOOST_CHECK(!destroyed); + // Second call should not decrement reference count 1 -> 0 + UnregisterAllValidationInterfaces(); + BOOST_CHECK(!destroyed); + }, + [&] { destroyed = true; })); + TestInterface::Call(); + BOOST_CHECK(destroyed); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 7b59d539a6..11c6bdad91 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> -#include <versionbits.h> -#include <test/util/setup_common.h> #include <chainparams.h> -#include <validation.h> #include <consensus/params.h> +#include <test/util/setup_common.h> +#include <validation.h> +#include <versionbits.h> #include <boost/test/unit_test.hpp> diff --git a/src/txdb.cpp b/src/txdb.cpp index acc47ab45e..071aa1336b 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 47b0d39ea4..102e50dc5c 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/undo.h b/src/undo.h index 80bbd146f3..a98f046735 100644 --- a/src/undo.h +++ b/src/undo.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index 60bd27bf90..5354cdb962 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/bip32.cpp b/src/util/bip32.cpp index 6f176dd5ec..4c7e948368 100644 --- a/src/util/bip32.cpp +++ b/src/util/bip32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/bip32.h b/src/util/bip32.h index 7e58b79f38..347e83db9e 100644 --- a/src/util/bip32.h +++ b/src/util/bip32.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index 544cfb58f9..1bc8d02eab 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/moneystr.h b/src/util/moneystr.h index d8b08adc24..9d2b6da0fc 100644 --- a/src/util/moneystr.h +++ b/src/util/moneystr.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 16917ccb47..3a903b6897 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 11317f0432..bd988f1410 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/string.h b/src/util/string.h index 694f0a1ca4..b8e2a06235 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/system.cpp b/src/util/system.cpp index b0a538b527..69a7be96dc 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -226,10 +226,11 @@ static bool CheckValid(const std::string& key, const util::SettingsValue& val, u return true; } -ArgsManager::ArgsManager() -{ - // nothing to do -} +// Define default constructor and destructor that are not inline, so code instantiating this class doesn't need to +// #include class definitions for all members. +// For example, m_settings has an internal dependency on univalue. +ArgsManager::ArgsManager() {} +ArgsManager::~ArgsManager() {} const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const { diff --git a/src/util/system.h b/src/util/system.h index 3138522b5c..96f51e6074 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -192,6 +192,7 @@ protected: public: ArgsManager(); + ~ArgsManager(); /** * Select the network in use diff --git a/src/util/time.cpp b/src/util/time.cpp index 0938ff36a6..e96972fe12 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/time.h b/src/util/time.h index 77de1e047d..b00c25f67c 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/url.h b/src/util/url.h index e9ea2ab765..5a7b11fa04 100644 --- a/src/util/url.h +++ b/src/util/url.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,8 @@ #include <string> -std::string urlDecode(const std::string &urlEncoded); +using UrlDecodeFn = std::string(const std::string& url_encoded); +UrlDecodeFn urlDecode; +extern UrlDecodeFn* const URL_DECODE; #endif // BITCOIN_UTIL_URL_H diff --git a/src/validation.cpp b/src/validation.cpp index 7ee94f8657..60d028336f 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -20,6 +20,7 @@ #include <index/txindex.h> #include <logging.h> #include <logging/timer.h> +#include <optional.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -76,20 +77,19 @@ bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIn return false; } -namespace { -BlockManager g_blockman; -} // anon namespace - -std::unique_ptr<CChainState> g_chainstate; +ChainstateManager g_chainman; -CChainState& ChainstateActive() { - assert(g_chainstate); - return *g_chainstate; +CChainState& ChainstateActive() +{ + LOCK(::cs_main); + assert(g_chainman.m_active_chainstate); + return *g_chainman.m_active_chainstate; } -CChain& ChainActive() { - assert(g_chainstate); - return g_chainstate->m_chain; +CChain& ChainActive() +{ + LOCK(::cs_main); + return ::ChainstateActive().m_chain; } /** @@ -151,8 +151,8 @@ namespace { CBlockIndex* LookupBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); - BlockMap::const_iterator it = g_blockman.m_block_index.find(hash); - return it == g_blockman.m_block_index.end() ? nullptr : it->second; + BlockMap::const_iterator it = g_chainman.BlockIndex().find(hash); + return it == g_chainman.BlockIndex().end() ? nullptr : it->second; } CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) @@ -1016,7 +1016,7 @@ bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs // scripts (ie, other policy checks pass). We perform the inexpensive // checks first and avoid hashing and signature verification unless those // checks pass, to mitigate CPU exhaustion denial-of-service attacks. - PrecomputedTransactionData txdata(*ptx); + PrecomputedTransactionData txdata; if (!PolicyScriptChecks(args, workspace, txdata)) return false; @@ -1242,10 +1242,9 @@ void CoinsViews::InitCache() m_cacheview = MakeUnique<CCoinsViewCache>(&m_catcherview); } -// NOTE: for now m_blockman is set to a global, but this will be changed -// in a future commit. -CChainState::CChainState() : m_blockman(g_blockman) {} - +CChainState::CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash) + : m_blockman(blockman), + m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( size_t cache_size_bytes, @@ -1253,6 +1252,10 @@ void CChainState::InitCoinsDB( bool should_wipe, std::string leveldb_name) { + if (!m_from_snapshot_blockhash.IsNull()) { + leveldb_name += "_" + m_from_snapshot_blockhash.ToString(); + } + m_coins_views = MakeUnique<CoinsViews>( leveldb_name, cache_size_bytes, in_memory, should_wipe); } @@ -1294,7 +1297,8 @@ static CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; BlockMap& BlockIndex() { - return g_blockman.m_block_index; + LOCK(::cs_main); + return g_chainman.m_blockman.m_block_index; } static void AlertNotify(const std::string& strMessage) @@ -1512,6 +1516,10 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const C return true; } + if (!txdata.m_ready) { + txdata.Init(tx); + } + for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; const Coin& coin = inputs.AccessCoin(prevout); @@ -2075,15 +2083,19 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockUndo blockundo; + // Precomputed transaction data pointers must not be invalidated + // until after `control` has run the script checks (potentially + // in multiple threads). Preallocate the vector size so a new allocation + // doesn't invalidate pointers into the vector, and keep txsdata in scope + // for as long as `control`. CCheckQueueControl<CScriptCheck> control(fScriptChecks && g_parallel_script_checks ? &scriptcheckqueue : nullptr); + std::vector<PrecomputedTransactionData> txsdata(block.vtx.size()); std::vector<int> prevheights; CAmount nFees = 0; int nInputs = 0; int64_t nSigOpsCost = 0; blockundo.vtxundo.reserve(block.vtx.size() - 1); - std::vector<PrecomputedTransactionData> txdata; - txdata.reserve(block.vtx.size()); // Required so that pointers to individual PrecomputedTransactionData don't get invalidated for (unsigned int i = 0; i < block.vtx.size(); i++) { const CTransaction &tx = *(block.vtx[i]); @@ -2130,13 +2142,12 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops"); } - txdata.emplace_back(tx); if (!tx.IsCoinBase()) { std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ TxValidationState tx_state; - if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { + if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txsdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { // Any transaction validation failure in ConnectBlock is a block consensus failure state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), tx_state.GetDebugMessage()); @@ -3443,7 +3454,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio if (fCheckpointsEnabled) { // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our - // g_blockman.m_block_index. + // BlockIndex(). CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); @@ -3651,7 +3662,8 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValid LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast - bool accepted = g_blockman.AcceptBlockHeader(header, state, chainparams, &pindex); + bool accepted = g_chainman.m_blockman.AcceptBlockHeader( + header, state, chainparams, &pindex); ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); if (!accepted) { @@ -3853,7 +3865,7 @@ void PruneOneBlockFile(const int fileNumber) { LOCK(cs_LastBlockFile); - for (const auto& entry : g_blockman.m_block_index) { + for (const auto& entry : g_chainman.BlockIndex()) { CBlockIndex* pindex = entry.second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; @@ -3867,12 +3879,12 @@ void PruneOneBlockFile(const int fileNumber) // to be downloaded again in order to consider its chain, at which // point it would be considered as a candidate for // m_blocks_unlinked or setBlockIndexCandidates. - auto range = g_blockman.m_blocks_unlinked.equal_range(pindex->pprev); + auto range = g_chainman.m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); while (range.first != range.second) { std::multimap<CBlockIndex *, CBlockIndex *>::iterator _it = range.first; range.first++; if (_it->second == pindex) { - g_blockman.m_blocks_unlinked.erase(_it); + g_chainman.m_blockman.m_blocks_unlinked.erase(_it); } } } @@ -4109,9 +4121,11 @@ void BlockManager::Unload() { bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - if (!g_blockman.LoadBlockIndex( - chainparams.GetConsensus(), *pblocktree, ::ChainstateActive().setBlockIndexCandidates)) + if (!g_chainman.m_blockman.LoadBlockIndex( + chainparams.GetConsensus(), *pblocktree, + ::ChainstateActive().setBlockIndexCandidates)) { return false; + } // Load block file info pblocktree->ReadLastBlockFile(nLastBlockFile); @@ -4133,7 +4147,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - for (const std::pair<const uint256, CBlockIndex*>& item : g_blockman.m_block_index) + for (const std::pair<const uint256, CBlockIndex*>& item : g_chainman.BlockIndex()) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { @@ -4510,26 +4524,15 @@ bool CChainState::RewindBlockIndex(const CChainParams& params) PruneBlockIndexCandidates(); CheckBlockIndex(params.GetConsensus()); - } - } - - return true; -} - -bool RewindBlockIndex(const CChainParams& params) { - if (!::ChainstateActive().RewindBlockIndex(params)) { - return false; - } - LOCK(cs_main); - if (::ChainActive().Tip() != nullptr) { - // FlushStateToDisk can possibly read ::ChainActive(). Be conservative - // and skip it here, we're about to -reindex-chainstate anyway, so - // it'll get called a bunch real soon. - BlockValidationState state; - if (!::ChainstateActive().FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { - LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); - return false; + // FlushStateToDisk can possibly read ::ChainActive(). Be conservative + // and skip it here, we're about to -reindex-chainstate anyway, so + // it'll get called a bunch real soon. + BlockValidationState state; + if (!FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { + LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); + return false; + } } } @@ -4547,8 +4550,7 @@ void CChainState::UnloadBlockIndex() { void UnloadBlockIndex() { LOCK(cs_main); - ::ChainActive().SetTip(nullptr); - g_blockman.Unload(); + g_chainman.Unload(); pindexBestInvalid = nullptr; pindexBestHeader = nullptr; mempool.clear(); @@ -4561,8 +4563,6 @@ void UnloadBlockIndex() warningcache[b].clear(); } fHavePruned = false; - - ::ChainstateActive().UnloadBlockIndex(); } bool LoadBlockIndex(const CChainParams& chainparams) @@ -4572,7 +4572,7 @@ bool LoadBlockIndex(const CChainParams& chainparams) if (!fReindex) { bool ret = LoadBlockIndexDB(chainparams); if (!ret) return false; - needs_init = g_blockman.m_block_index.empty(); + needs_init = g_chainman.m_blockman.m_block_index.empty(); } if (needs_init) { @@ -4693,7 +4693,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // Activate the genesis block so normal node progress can continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { + if (!ActivateBestChain(state, chainparams, nullptr)) { break; } } @@ -4923,6 +4923,14 @@ void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) assert(nNodes == forward.size()); } +std::string CChainState::ToString() +{ + CBlockIndex* tip = m_chain.Tip(); + return strprintf("Chainstate [%s] @ height %d (%s)", + m_from_snapshot_blockhash.IsNull() ? "ibd" : "snapshot", + tip ? tip->nHeight : -1, tip ? tip->GetBlockHash().ToString() : "null"); +} + std::string CBlockFileInfo::ToString() const { return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); @@ -5110,10 +5118,99 @@ public: CMainCleanup() {} ~CMainCleanup() { // block headers - BlockMap::iterator it1 = g_blockman.m_block_index.begin(); - for (; it1 != g_blockman.m_block_index.end(); it1++) + BlockMap::iterator it1 = g_chainman.BlockIndex().begin(); + for (; it1 != g_chainman.BlockIndex().end(); it1++) delete (*it1).second; - g_blockman.m_block_index.clear(); + g_chainman.BlockIndex().clear(); } }; static CMainCleanup instance_of_cmaincleanup; + +Optional<uint256> ChainstateManager::SnapshotBlockhash() const { + if (m_active_chainstate != nullptr) { + // If a snapshot chainstate exists, it will always be our active. + return m_active_chainstate->m_from_snapshot_blockhash; + } + return {}; +} + +std::vector<CChainState*> ChainstateManager::GetAll() +{ + std::vector<CChainState*> out; + + if (!IsSnapshotValidated() && m_ibd_chainstate) { + out.push_back(m_ibd_chainstate.get()); + } + + if (m_snapshot_chainstate) { + out.push_back(m_snapshot_chainstate.get()); + } + + return out; +} + +CChainState& ChainstateManager::InitializeChainstate(const uint256& snapshot_blockhash) +{ + bool is_snapshot = !snapshot_blockhash.IsNull(); + std::unique_ptr<CChainState>& to_modify = + is_snapshot ? m_snapshot_chainstate : m_ibd_chainstate; + + if (to_modify) { + throw std::logic_error("should not be overwriting a chainstate"); + } + + to_modify.reset(new CChainState(m_blockman, snapshot_blockhash)); + + // Snapshot chainstates and initial IBD chaintates always become active. + if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { + LogPrintf("Switching active chainstate to %s\n", to_modify->ToString()); + m_active_chainstate = to_modify.get(); + } else { + throw std::logic_error("unexpected chainstate activation"); + } + + return *to_modify; +} + +CChain& ChainstateManager::ActiveChain() const +{ + assert(m_active_chainstate); + return m_active_chainstate->m_chain; +} + +bool ChainstateManager::IsSnapshotActive() const +{ + return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); +} + +CChainState& ChainstateManager::ValidatedChainstate() const +{ + if (m_snapshot_chainstate && IsSnapshotValidated()) { + return *m_snapshot_chainstate.get(); + } + assert(m_ibd_chainstate); + return *m_ibd_chainstate.get(); +} + +bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const +{ + return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); +} + +void ChainstateManager::Unload() +{ + for (CChainState* chainstate : this->GetAll()) { + chainstate->m_chain.SetTip(nullptr); + chainstate->UnloadBlockIndex(); + } + + m_blockman.Unload(); +} + +void ChainstateManager::Reset() +{ + m_ibd_chainstate.reset(); + m_snapshot_chainstate.reset(); + m_active_chainstate = nullptr; + m_snapshot_validated = false; +} diff --git a/src/validation.h b/src/validation.h index a5335edc43..dbf7aa28db 100644 --- a/src/validation.h +++ b/src/validation.h @@ -14,6 +14,7 @@ #include <coins.h> #include <crypto/common.h> // for ReadLE64 #include <fs.h> +#include <optional.h> #include <policy/feerate.h> #include <protocol.h> // For CMessageHeader::MessageStartChars #include <script/script_error.h> @@ -379,9 +380,6 @@ bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainpar * Note that transaction witness validation rules are always enforced when P2SH is enforced. */ bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& params); -/** When there are blocks in the active chain with missing data, rewind the chainstate and remove them from the block index */ -bool RewindBlockIndex(const CChainParams& params) LOCKS_EXCLUDED(cs_main); - /** Compute at which vout of the block's coinbase transaction the witness commitment occurs, or -1 if not found */ int GetWitnessCommitmentIndex(const CBlock& block); @@ -539,6 +537,9 @@ enum class CoinsCacheSizeState OK = 0 }; +// Defined below, but needed for `friend` usage in CChainState. +class ChainstateManager; + /** * CChainState stores and provides an API to update our local knowledge of the * current best chain. @@ -591,8 +592,7 @@ private: std::unique_ptr<CoinsViews> m_coins_views; public: - CChainState(BlockManager& blockman) : m_blockman(blockman) {} - CChainState(); + explicit CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash = uint256()); /** * Initialize the CoinsViews UTXO set database management data structures. The in-memory @@ -621,6 +621,13 @@ public: CChain m_chain; /** + * The blockhash which is the base of the snapshot this chainstate was created from. + * + * IsNull() if this chainstate was not created from a snapshot. + */ + const uint256 m_from_snapshot_blockhash{}; + + /** * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be * missing the data for the block. @@ -741,6 +748,8 @@ public: size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + std::string ToString() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + private: bool ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); bool ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); @@ -753,6 +762,8 @@ private: //! Mark a block as not having block data void EraseBlockData(CBlockIndex* index) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + friend ChainstateManager; }; /** Mark a block as precious and reorganize. @@ -768,6 +779,150 @@ bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparam /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +/** + * Provides an interface for creating and interacting with one or two + * chainstates: an IBD chainstate generated by downloading blocks, and + * an optional snapshot chainstate loaded from a UTXO snapshot. Managed + * chainstates can be maintained at different heights simultaneously. + * + * This class provides abstractions that allow the retrieval of the current + * most-work chainstate ("Active") as well as chainstates which may be in + * background use to validate UTXO snapshots. + * + * Definitions: + * + * *IBD chainstate*: a chainstate whose current state has been "fully" + * validated by the initial block download process. + * + * *Snapshot chainstate*: a chainstate populated by loading in an + * assumeutxo UTXO snapshot. + * + * *Active chainstate*: the chainstate containing the current most-work + * chain. Consulted by most parts of the system (net_processing, + * wallet) as a reflection of the current chain and UTXO set. + * This may either be an IBD chainstate or a snapshot chainstate. + * + * *Background IBD chainstate*: an IBD chainstate for which the + * IBD process is happening in the background while use of the + * active (snapshot) chainstate allows the rest of the system to function. + * + * *Validated chainstate*: the most-work chainstate which has been validated + * locally via initial block download. This will be the snapshot chainstate + * if a snapshot was loaded and all blocks up to the snapshot starting point + * have been downloaded and validated (via background validation), otherwise + * it will be the IBD chainstate. + */ +class ChainstateManager +{ +private: + //! The chainstate used under normal operation (i.e. "regular" IBD) or, if + //! a snapshot is in use, for background validation. + //! + //! Its contents (including on-disk data) will be deleted *upon shutdown* + //! after background validation of the snapshot has completed. We do not + //! free the chainstate contents immediately after it finishes validation + //! to cautiously avoid a case where some other part of the system is still + //! using this pointer (e.g. net_processing). + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + std::unique_ptr<CChainState> m_ibd_chainstate; + + //! A chainstate initialized on the basis of a UTXO snapshot. If this is + //! non-null, it is always our active chainstate. + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + std::unique_ptr<CChainState> m_snapshot_chainstate; + + //! Points to either the ibd or snapshot chainstate; indicates our + //! most-work chain. + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + CChainState* m_active_chainstate{nullptr}; + + //! If true, the assumed-valid chainstate has been fully validated + //! by the background validation chainstate. + bool m_snapshot_validated{false}; + + // For access to m_active_chainstate. + friend CChainState& ChainstateActive(); + friend CChain& ChainActive(); + +public: + //! A single BlockManager instance is shared across each constructed + //! chainstate to avoid duplicating block metadata. + BlockManager m_blockman GUARDED_BY(::cs_main); + + //! Instantiate a new chainstate and assign it based upon whether it is + //! from a snapshot. + //! + //! @param[in] snapshot_blockhash If given, signify that this chainstate + //! is based on a snapshot. + CChainState& InitializeChainstate(const uint256& snapshot_blockhash = uint256()) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Get all chainstates currently being used. + std::vector<CChainState*> GetAll(); + + //! The most-work chain. + CChain& ActiveChain() const; + int ActiveHeight() const { return ActiveChain().Height(); } + CBlockIndex* ActiveTip() const { return ActiveChain().Tip(); } + + BlockMap& BlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + { + return m_blockman.m_block_index; + } + + bool IsSnapshotActive() const; + + Optional<uint256> SnapshotBlockhash() const; + + //! Is there a snapshot in use and has it been fully validated? + bool IsSnapshotValidated() const { return m_snapshot_validated; } + + //! @returns true if this chainstate is being used to validate an active + //! snapshot in the background. + bool IsBackgroundIBD(CChainState* chainstate) const; + + //! Return the most-work chainstate that has been fully validated. + //! + //! During background validation of a snapshot, this is the IBD chain. After + //! background validation has completed, this is the snapshot chain. + CChainState& ValidatedChainstate() const; + + CChain& ValidatedChain() const { return ValidatedChainstate().m_chain; } + CBlockIndex* ValidatedTip() const { return ValidatedChain().Tip(); } + + //! Unload block index and chain data before shutdown. + void Unload() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Clear (deconstruct) chainstate data. + void Reset(); +}; + +extern ChainstateManager g_chainman GUARDED_BY(::cs_main); + /** @returns the most-work valid chainstate. */ CChainState& ChainstateActive(); @@ -777,11 +932,6 @@ CChain& ChainActive(); /** @returns the global block index map. */ BlockMap& BlockIndex(); -// Most often ::ChainstateActive() should be used instead of this, but some code -// may not be able to assume that this has been initialized yet and so must use it -// directly, e.g. init.cpp. -extern std::unique_ptr<CChainState> g_chainstate; - /** Global variable that points to the active block tree (protected by cs_main) */ extern std::unique_ptr<CBlockTreeDB> pblocktree; diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index f9f61e8a09..11000774c0 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -16,36 +16,75 @@ #include <unordered_map> #include <utility> -#include <boost/signals2/signal.hpp> - -struct ValidationInterfaceConnections { - boost::signals2::scoped_connection UpdatedBlockTip; - boost::signals2::scoped_connection TransactionAddedToMempool; - boost::signals2::scoped_connection BlockConnected; - boost::signals2::scoped_connection BlockDisconnected; - boost::signals2::scoped_connection TransactionRemovedFromMempool; - boost::signals2::scoped_connection ChainStateFlushed; - boost::signals2::scoped_connection BlockChecked; - boost::signals2::scoped_connection NewPoWValidBlock; -}; - +//! The MainSignalsInstance manages a list of shared_ptr<CValidationInterface> +//! callbacks. +//! +//! A std::unordered_map is used to track what callbacks are currently +//! registered, and a std::list is to used to store the callbacks that are +//! currently registered as well as any callbacks that are just unregistered +//! and about to be deleted when they are done executing. struct MainSignalsInstance { - boost::signals2::signal<void (const CBlockIndex *, const CBlockIndex *, bool fInitialDownload)> UpdatedBlockTip; - boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; - boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex)> BlockConnected; - boost::signals2::signal<void (const std::shared_ptr<const CBlock>&, const CBlockIndex* pindex)> BlockDisconnected; - boost::signals2::signal<void (const CTransactionRef &)> TransactionRemovedFromMempool; - boost::signals2::signal<void (const CBlockLocator &)> ChainStateFlushed; - boost::signals2::signal<void (const CBlock&, const BlockValidationState&)> BlockChecked; - boost::signals2::signal<void (const CBlockIndex *, const std::shared_ptr<const CBlock>&)> NewPoWValidBlock; - +private: + Mutex m_mutex; + //! List entries consist of a callback pointer and reference count. The + //! count is equal to the number of current executions of that entry, plus 1 + //! if it's registered. It cannot be 0 because that would imply it is + //! unregistered and also not being executed (so shouldn't exist). + struct ListEntry { std::shared_ptr<CValidationInterface> callbacks; int count = 1; }; + std::list<ListEntry> m_list GUARDED_BY(m_mutex); + std::unordered_map<CValidationInterface*, std::list<ListEntry>::iterator> m_map GUARDED_BY(m_mutex); + +public: // We are not allowed to assume the scheduler only runs in one thread, // but must ensure all callbacks happen in-order, so we end up creating // our own queue here :( SingleThreadedSchedulerClient m_schedulerClient; - std::unordered_map<CValidationInterface*, ValidationInterfaceConnections> m_connMainSignals; explicit MainSignalsInstance(CScheduler *pscheduler) : m_schedulerClient(pscheduler) {} + + void Register(std::shared_ptr<CValidationInterface> callbacks) + { + LOCK(m_mutex); + auto inserted = m_map.emplace(callbacks.get(), m_list.end()); + if (inserted.second) inserted.first->second = m_list.emplace(m_list.end()); + inserted.first->second->callbacks = std::move(callbacks); + } + + void Unregister(CValidationInterface* callbacks) + { + LOCK(m_mutex); + auto it = m_map.find(callbacks); + if (it != m_map.end()) { + if (!--it->second->count) m_list.erase(it->second); + m_map.erase(it); + } + } + + //! Clear unregisters every previously registered callback, erasing every + //! map entry. After this call, the list may still contain callbacks that + //! are currently executing, but it will be cleared when they are done + //! executing. + void Clear() + { + LOCK(m_mutex); + for (const auto& entry : m_map) { + if (!--entry.second->count) m_list.erase(entry.second); + } + m_map.clear(); + } + + template<typename F> void Iterate(F&& f) + { + WAIT_LOCK(m_mutex, lock); + for (auto it = m_list.begin(); it != m_list.end();) { + ++it->count; + { + REVERSE_LOCK(lock); + f(*it->callbacks); + } + it = --it->count ? std::next(it) : m_list.erase(it); + } + } }; static CMainSignals g_signals; @@ -78,15 +117,7 @@ CMainSignals& GetMainSignals() void RegisterSharedValidationInterface(std::shared_ptr<CValidationInterface> pwalletIn) { // Each connection captures pwalletIn to ensure that each callback is // executed before pwalletIn is destroyed. For more details see #18338. - ValidationInterfaceConnections& conns = g_signals.m_internals->m_connMainSignals[pwalletIn.get()]; - conns.UpdatedBlockTip = g_signals.m_internals->UpdatedBlockTip.connect(std::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); - conns.TransactionAddedToMempool = g_signals.m_internals->TransactionAddedToMempool.connect(std::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, std::placeholders::_1)); - conns.BlockConnected = g_signals.m_internals->BlockConnected.connect(std::bind(&CValidationInterface::BlockConnected, pwalletIn, std::placeholders::_1, std::placeholders::_2)); - conns.BlockDisconnected = g_signals.m_internals->BlockDisconnected.connect(std::bind(&CValidationInterface::BlockDisconnected, pwalletIn, std::placeholders::_1, std::placeholders::_2)); - conns.TransactionRemovedFromMempool = g_signals.m_internals->TransactionRemovedFromMempool.connect(std::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, std::placeholders::_1)); - conns.ChainStateFlushed = g_signals.m_internals->ChainStateFlushed.connect(std::bind(&CValidationInterface::ChainStateFlushed, pwalletIn, std::placeholders::_1)); - conns.BlockChecked = g_signals.m_internals->BlockChecked.connect(std::bind(&CValidationInterface::BlockChecked, pwalletIn, std::placeholders::_1, std::placeholders::_2)); - conns.NewPoWValidBlock = g_signals.m_internals->NewPoWValidBlock.connect(std::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, std::placeholders::_1, std::placeholders::_2)); + g_signals.m_internals->Register(std::move(pwalletIn)); } void RegisterValidationInterface(CValidationInterface* callbacks) @@ -103,7 +134,7 @@ void UnregisterSharedValidationInterface(std::shared_ptr<CValidationInterface> c void UnregisterValidationInterface(CValidationInterface* pwalletIn) { if (g_signals.m_internals) { - g_signals.m_internals->m_connMainSignals.erase(pwalletIn); + g_signals.m_internals->Unregister(pwalletIn); } } @@ -111,7 +142,7 @@ void UnregisterAllValidationInterfaces() { if (!g_signals.m_internals) { return; } - g_signals.m_internals->m_connMainSignals.clear(); + g_signals.m_internals->Clear(); } void CallFunctionInValidationInterfaceQueue(std::function<void ()> func) { @@ -151,7 +182,7 @@ void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockInd // in the same critical section where the chain is updated auto event = [pindexNew, pindexFork, fInitialDownload, this] { - m_internals->UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: new block hash=%s fork block hash=%s (in IBD=%s)", __func__, pindexNew->GetBlockHash().ToString(), @@ -161,7 +192,7 @@ void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockInd void CMainSignals::TransactionAddedToMempool(const CTransactionRef &ptx) { auto event = [ptx, this] { - m_internals->TransactionAddedToMempool(ptx); + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionAddedToMempool(ptx); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, ptx->GetHash().ToString(), @@ -170,7 +201,7 @@ void CMainSignals::TransactionAddedToMempool(const CTransactionRef &ptx) { void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef &ptx) { auto event = [ptx, this] { - m_internals->TransactionRemovedFromMempool(ptx); + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionRemovedFromMempool(ptx); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, ptx->GetHash().ToString(), @@ -179,7 +210,7 @@ void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef &ptx) { void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex) { auto event = [pblock, pindex, this] { - m_internals->BlockConnected(pblock, pindex); + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.BlockConnected(pblock, pindex); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s block height=%d", __func__, pblock->GetHash().ToString(), @@ -189,7 +220,7 @@ void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, c void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) { auto event = [pblock, pindex, this] { - m_internals->BlockDisconnected(pblock, pindex); + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.BlockDisconnected(pblock, pindex); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s block height=%d", __func__, pblock->GetHash().ToString(), @@ -198,7 +229,7 @@ void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock void CMainSignals::ChainStateFlushed(const CBlockLocator &locator) { auto event = [locator, this] { - m_internals->ChainStateFlushed(locator); + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.ChainStateFlushed(locator); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s", __func__, locator.IsNull() ? "null" : locator.vHave.front().ToString()); @@ -207,10 +238,10 @@ void CMainSignals::ChainStateFlushed(const CBlockLocator &locator) { void CMainSignals::BlockChecked(const CBlock& block, const BlockValidationState& state) { LOG_EVENT("%s: block hash=%s state=%s", __func__, block.GetHash().ToString(), state.ToString()); - m_internals->BlockChecked(block, state); + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.BlockChecked(block, state); }); } void CMainSignals::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock> &block) { LOG_EVENT("%s: block hash=%s", __func__, block->GetHash().ToString()); - m_internals->NewPoWValidBlock(pindex, block); + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.NewPoWValidBlock(pindex, block); }); } diff --git a/src/validationinterface.h b/src/validationinterface.h index f9a359b7ad..cb0204a555 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -171,9 +171,7 @@ protected: * Notifies listeners that a block which builds directly on our current tip * has been received and connected to the headers tree, though not validated yet */ virtual void NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& block) {}; - friend void ::RegisterSharedValidationInterface(std::shared_ptr<CValidationInterface>); - friend void ::UnregisterValidationInterface(CValidationInterface*); - friend void ::UnregisterAllValidationInterfaces(); + friend class CMainSignals; }; struct MainSignalsInstance; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 5bbb2c0ad0..079a5d3d53 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -106,6 +106,9 @@ bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& target_v best_selection = curr_selection; best_selection.resize(utxo_pool.size()); best_waste = curr_waste; + if (best_waste == 0) { + break; + } } curr_waste -= (curr_value - actual_target); // Remove the excess value as we will be selecting different coins now backtrack = true; diff --git a/src/wallet/db.h b/src/wallet/db.h index bebaa55d05..1c5f42abca 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 1623ab9074..dafa022ded 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index 859f754761..fc038ae731 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 50f064b305..8688f3df5e 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -57,7 +57,6 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-salvagewallet", "Attempt to recover private keys from a corrupt wallet on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-spendzeroconfchange", strprintf("Spend unconfirmed change when sending transactions (default: %u)", DEFAULT_SPEND_ZEROCONF_CHANGE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-txconfirmtarget=<n>", strprintf("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)", DEFAULT_TX_CONFIRM_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-upgradewallet", "Upgrade wallet to latest format on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-wallet=<path>", "Specify wallet database path. Can be specified multiple times to load multiple wallets. Path is interpreted relative to <walletdir> if it is not absolute, and will be created if it does not exist (as a directory containing a wallet.dat file and log files). For backwards compatibility this will also accept names of existing data files in <walletdir>.)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); gArgs.AddArg("-walletbroadcast", strprintf("Make the wallet broadcast transactions (default: %u)", DEFAULT_WALLETBROADCAST), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-walletdir=<dir>", "Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); @@ -116,12 +115,6 @@ bool WalletInit::ParameterInteraction() const } } - if (is_multiwallet) { - if (gArgs.GetBoolArg("-upgradewallet", false)) { - return InitError(strprintf("%s is only allowed with a single wallet file", "-upgradewallet")); - } - } - if (gArgs.GetBoolArg("-sysperms", false)) return InitError("-sysperms is not allowed in combination with enabled wallet functionality"); diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index d6e44c7be5..0afd2dfcf0 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index e4d0a3fa6d..ceb7a7d287 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,6 +28,8 @@ +using interfaces::FoundBlock; + std::string static EncodeDumpString(const std::string &str) { std::stringstream ret; for (const unsigned char c : str) { @@ -60,12 +62,13 @@ static bool GetWalletAddressesForKey(LegacyScriptPubKeyMan* spk_man, const CWall CKey key; spk_man->GetKey(keyid, key); for (const auto& dest : GetAllDestinationsForKey(key.GetPubKey())) { - if (pwallet->mapAddressBook.count(dest)) { + const auto* address_book_entry = pwallet->FindAddressBookEntry(dest); + if (address_book_entry) { if (!strAddr.empty()) { strAddr += ","; } strAddr += EncodeDestination(dest); - strLabel = EncodeDumpString(pwallet->mapAddressBook.at(dest).name); + strLabel = EncodeDumpString(address_book_entry->GetLabel()); fLabelFound = true; } } @@ -127,7 +130,7 @@ UniValue importprivkey(const JSONRPCRequest& request) EnsureLegacyScriptPubKeyMan(*wallet, true); - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); bool fRescan = true; { auto locked_chain = pwallet->chain().lock(); @@ -168,7 +171,7 @@ UniValue importprivkey(const JSONRPCRequest& request) // label all new addresses, and label existing addresses if a // label was passed. for (const auto& dest : GetAllDestinationsForKey(pubkey)) { - if (!request.params[1].isNull() || pwallet->mapAddressBook.count(dest) == 0) { + if (!request.params[1].isNull() || !pwallet->FindAddressBookEntry(dest)) { pwallet->SetAddressBook(dest, strLabel, "receive"); } } @@ -271,7 +274,7 @@ UniValue importaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -359,8 +362,9 @@ UniValue importprunedfunds(const JSONRPCRequest& request) } auto locked_chain = pwallet->chain().lock(); - Optional<int> height = locked_chain->getBlockHeight(merkleBlock.header.GetHash()); - if (height == nullopt) { + LOCK(pwallet->cs_wallet); + int height; + if (!pwallet->chain().findAncestorByHash(pwallet->GetLastBlockHash(), merkleBlock.header.GetHash(), FoundBlock().height(height))) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); } @@ -371,11 +375,9 @@ UniValue importprunedfunds(const JSONRPCRequest& request) unsigned int txnIndex = vIndex[it - vMatch.begin()]; - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *height, merkleBlock.header.GetHash(), txnIndex); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, height, merkleBlock.header.GetHash(), txnIndex); wtx.m_confirm = confirm; - LOCK(pwallet->cs_wallet); - if (pwallet->IsMine(*wtx.tx)) { pwallet->AddToWallet(wtx, false); return NullUniValue; @@ -472,7 +474,7 @@ UniValue importpubkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -547,7 +549,7 @@ UniValue importwallet(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -565,8 +567,7 @@ UniValue importwallet(const JSONRPCRequest& request) if (!file.is_open()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); } - Optional<int> tip_height = locked_chain->getHeight(); - nTimeBegin = tip_height ? locked_chain->getBlockTime(*tip_height) : 0; + CHECK_NONFATAL(pwallet->chain().findBlock(pwallet->GetLastBlockHash(), FoundBlock().time(nTimeBegin))); int64_t nFilesize = std::max((int64_t)1, (int64_t)file.tellg()); file.seekg(0, file.beg); @@ -790,9 +791,10 @@ UniValue dumpwallet(const JSONRPCRequest& request) // produce output file << strprintf("# Wallet dump created by Bitcoin %s\n", CLIENT_BUILD); file << strprintf("# * Created on %s\n", FormatISO8601DateTime(GetTime())); - const Optional<int> tip_height = locked_chain->getHeight(); - file << strprintf("# * Best block at time of backup was %i (%s),\n", tip_height.get_value_or(-1), tip_height ? locked_chain->getBlockHash(*tip_height).ToString() : "(missing block hash)"); - file << strprintf("# mined on %s\n", tip_height ? FormatISO8601DateTime(locked_chain->getBlockTime(*tip_height)) : "(missing block time)"); + file << strprintf("# * Best block at time of backup was %i (%s),\n", pwallet->GetLastBlockHeight(), pwallet->GetLastBlockHash().ToString()); + int64_t block_time = 0; + CHECK_NONFATAL(pwallet->chain().findBlock(pwallet->GetLastBlockHash(), FoundBlock().time(block_time))); + file << strprintf("# mined on %s\n", FormatISO8601DateTime(block_time)); file << "\n"; // add the base58check encoded extended master if the wallet uses HD @@ -1363,7 +1365,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -1378,20 +1380,13 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) EnsureWalletIsUnlocked(pwallet); // Verify all timestamps are present before importing any keys. - const Optional<int> tip_height = locked_chain->getHeight(); - now = tip_height ? locked_chain->getBlockMedianTimePast(*tip_height) : 0; + CHECK_NONFATAL(pwallet->chain().findBlock(pwallet->GetLastBlockHash(), FoundBlock().time(nLowestTimestamp).mtpTime(now))); for (const UniValue& data : requests.getValues()) { GetImportTimestamp(data, now); } const int64_t minimumTimestamp = 1; - if (fRescan && tip_height) { - nLowestTimestamp = locked_chain->getBlockTime(*tip_height); - } else { - fRescan = false; - } - for (const UniValue& data : requests.getValues()) { const int64_t timestamp = std::max(GetImportTimestamp(data, now), minimumTimestamp); const UniValue result = ProcessImport(pwallet, data, timestamp); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index d8b3b57b61..46efb18793 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -37,6 +37,8 @@ #include <univalue.h> +using interfaces::FoundBlock; + static const std::string WALLET_ENDPOINT_BASE = "/wallet/"; static const std::string HELP_REQUIRING_PASSPHRASE{"\nRequires wallet passphrase to be set with walletpassphrase call if wallet is encrypted.\n"}; @@ -78,9 +80,9 @@ bool HaveKey(const SigningProvider& wallet, const CKey& key) bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& wallet_name) { - if (request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { + if (URL_DECODE && request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { // wallet endpoint was used - wallet_name = urlDecode(request.URI.substr(WALLET_ENDPOINT_BASE.size())); + wallet_name = URL_DECODE(request.URI.substr(WALLET_ENDPOINT_BASE.size())); return true; } return false; @@ -143,8 +145,7 @@ static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& lo entry.pushKV("blockheight", wtx.m_confirm.block_height); entry.pushKV("blockindex", wtx.m_confirm.nIndex); int64_t block_time; - bool found_block = chain.findBlock(wtx.m_confirm.hashBlock, nullptr /* block */, &block_time); - CHECK_NONFATAL(found_block); + CHECK_NONFATAL(chain.findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(block_time))); entry.pushKV("blocktime", block_time); } else { entry.pushKV("trusted", wtx.IsTrusted(locked_chain)); @@ -499,8 +500,9 @@ static UniValue listaddressgroupings(const JSONRPCRequest& request) addressInfo.push_back(EncodeDestination(address)); addressInfo.push_back(ValueFromAmount(balances[address])); { - if (pwallet->mapAddressBook.find(address) != pwallet->mapAddressBook.end()) { - addressInfo.push_back(pwallet->mapAddressBook.find(address)->second.name); + const auto* address_book_entry = pwallet->FindAddressBookEntry(address); + if (address_book_entry) { + addressInfo.push_back(address_book_entry->GetLabel()); } } jsonGrouping.push_back(addressInfo); @@ -570,6 +572,52 @@ static UniValue signmessage(const JSONRPCRequest& request) return signature; } +static CAmount GetReceived(interfaces::Chain::Lock& locked_chain, const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +{ + std::set<CTxDestination> address_set; + + if (by_label) { + // Get the set of addresses assigned to label + std::string label = LabelFromValue(params[0]); + address_set = wallet.GetLabelAddresses(label); + } else { + // Get the address + CTxDestination dest = DecodeDestination(params[0].get_str()); + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); + } + CScript script_pub_key = GetScriptForDestination(dest); + if (!wallet.IsMine(script_pub_key)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); + } + address_set.insert(dest); + } + + // Minimum confirmations + int min_depth = 1; + if (!params[1].isNull()) + min_depth = params[1].get_int(); + + // Tally + CAmount amount = 0; + for (const std::pair<const uint256, CWalletTx>& wtx_pair : wallet.mapWallet) { + const CWalletTx& wtx = wtx_pair.second; + if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx) || wtx.GetDepthInMainChain() < min_depth) { + continue; + } + + for (const CTxOut& txout : wtx.tx->vout) { + CTxDestination address; + if (ExtractDestination(txout.scriptPubKey, address) && wallet.IsMine(address) && address_set.count(address)) { + amount += txout.nValue; + } + } + } + + return amount; +} + + static UniValue getreceivedbyaddress(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -607,36 +655,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - // Bitcoin address - CTxDestination dest = DecodeDestination(request.params[0].get_str()); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); - } - CScript scriptPubKey = GetScriptForDestination(dest); - if (!pwallet->IsMine(scriptPubKey)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); - } - - // Minimum confirmations - int nMinDepth = 1; - if (!request.params[1].isNull()) - nMinDepth = request.params[1].get_int(); - - // Tally - CAmount nAmount = 0; - for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { - continue; - } - - for (const CTxOut& txout : wtx.tx->vout) - if (txout.scriptPubKey == scriptPubKey) - if (wtx.GetDepthInMainChain() >= nMinDepth) - nAmount += txout.nValue; - } - - return ValueFromAmount(nAmount); + return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ false)); } @@ -677,34 +696,7 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - // Minimum confirmations - int nMinDepth = 1; - if (!request.params[1].isNull()) - nMinDepth = request.params[1].get_int(); - - // Get the set of pub keys assigned to label - std::string label = LabelFromValue(request.params[0]); - std::set<CTxDestination> setAddress = pwallet->GetLabelAddresses(label); - - // Tally - CAmount nAmount = 0; - for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { - continue; - } - - for (const CTxOut& txout : wtx.tx->vout) - { - CTxDestination address; - if (ExtractDestination(txout.scriptPubKey, address) && pwallet->IsMine(address) && setAddress.count(address)) { - if (wtx.GetDepthInMainChain() >= nMinDepth) - nAmount += txout.nValue; - } - } - } - - return ValueFromAmount(nAmount); + return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ true)); } @@ -731,9 +723,9 @@ static UniValue getbalance(const JSONRPCRequest& request) RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " received for this wallet." }, RPCExamples{ - "\nThe total amount in the wallet with 1 or more confirmations\n" + "\nThe total amount in the wallet with 0 or more confirmations\n" + HelpExampleCli("getbalance", "") + - "\nThe total amount in the wallet at least 6 blocks confirmed\n" + "\nThe total amount in the wallet with at least 6 confirmations\n" + HelpExampleCli("getbalance", "\"*\" 6") + "\nAs a JSON-RPC call\n" + HelpExampleRpc("getbalance", "\"*\", 6") @@ -1092,13 +1084,13 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, const CWalle UniValue ret(UniValue::VARR); std::map<std::string, tallyitem> label_tally; - // Create mapAddressBook iterator + // Create m_address_book iterator // If we aren't filtering, go from begin() to end() - auto start = pwallet->mapAddressBook.begin(); - auto end = pwallet->mapAddressBook.end(); + auto start = pwallet->m_address_book.begin(); + auto end = pwallet->m_address_book.end(); // If we are filtering, find() the applicable entry if (has_filtered_address) { - start = pwallet->mapAddressBook.find(filtered_address); + start = pwallet->m_address_book.find(filtered_address); if (start != end) { end = std::next(start); } @@ -1106,8 +1098,9 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, const CWalle for (auto item_it = start; item_it != end; ++item_it) { + if (item_it->second.IsChange()) continue; const CTxDestination& address = item_it->first; - const std::string& label = item_it->second.name; + const std::string& label = item_it->second.GetLabel(); auto it = mapTally.find(address); if (it == mapTally.end() && !fIncludeEmpty) continue; @@ -1307,8 +1300,9 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, const CWalle MaybePushAddress(entry, s.destination); entry.pushKV("category", "send"); entry.pushKV("amount", ValueFromAmount(-s.amount)); - if (pwallet->mapAddressBook.count(s.destination)) { - entry.pushKV("label", pwallet->mapAddressBook.at(s.destination).name); + const auto* address_book_entry = pwallet->FindAddressBookEntry(s.destination); + if (address_book_entry) { + entry.pushKV("label", address_book_entry->GetLabel()); } entry.pushKV("vout", s.vout); entry.pushKV("fee", ValueFromAmount(-nFee)); @@ -1324,8 +1318,9 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, const CWalle for (const COutputEntry& r : listReceived) { std::string label; - if (pwallet->mapAddressBook.count(r.destination)) { - label = pwallet->mapAddressBook.at(r.destination).name; + const auto* address_book_entry = pwallet->FindAddressBookEntry(r.destination); + if (address_book_entry) { + label = address_book_entry->GetLabel(); } if (filter_label && label != *filter_label) { continue; @@ -1349,7 +1344,7 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, const CWalle entry.pushKV("category", "receive"); } entry.pushKV("amount", ValueFromAmount(r.amount)); - if (pwallet->mapAddressBook.count(r.destination)) { + if (address_book_entry) { entry.pushKV("label", label); } entry.pushKV("vout", r.vout); @@ -1395,8 +1390,8 @@ UniValue listtransactions(const JSONRPCRequest& request) "\nIf a label name is provided, this will return only incoming transactions paying to addresses with the specified label.\n" "\nReturns up to 'count' most recent transactions skipping the first 'from' transactions.\n", { - {"label", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If set, should be a valid label name to return only incoming transactions\n" - " with the specified label, or \"*\" to disable filtering and return all transactions."}, + {"label|dummy", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If set, should be a valid label name to return only incoming transactions\n" + "with the specified label, or \"*\" to disable filtering and return all transactions."}, {"count", RPCArg::Type::NUM, /* default */ "10", "The number of transactions to return"}, {"skip", RPCArg::Type::NUM, /* default */ "0", "The number of transactions to skip"}, {"include_watchonly", RPCArg::Type::BOOL, /* default */ "true for watch-only wallets, otherwise false", "Include transactions to watch-only addresses (see 'importaddress')"}, @@ -1574,8 +1569,9 @@ static UniValue listsinceblock(const JSONRPCRequest& request) uint256 blockId; if (!request.params[0].isNull() && !request.params[0].get_str().empty()) { blockId = ParseHashV(request.params[0], "blockhash"); - height = locked_chain->findFork(blockId, &altheight); - if (!height) { + height.emplace(); + altheight.emplace(); + if (!pwallet->chain().findCommonAncestor(blockId, pwallet->GetLastBlockHash(), /* ancestor out */ FoundBlock().height(*height), /* blockId out */ FoundBlock().height(*altheight))) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } } @@ -1594,8 +1590,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) bool include_removed = (request.params[3].isNull() || request.params[3].get_bool()); - const Optional<int> tip_height = locked_chain->getHeight(); - int depth = tip_height && height ? (1 + *tip_height - *height) : -1; + int depth = height ? pwallet->GetLastBlockHeight() + 1 - *height : -1; UniValue transactions(UniValue::VARR); @@ -1612,7 +1607,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) UniValue removed(UniValue::VARR); while (include_removed && altheight && *altheight > *height) { CBlock block; - if (!pwallet->chain().findBlock(blockId, &block) || block.IsNull()) { + if (!pwallet->chain().findBlock(blockId, FoundBlock().data(block)) || block.IsNull()) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); } for (const CTransactionRef& tx : block.vtx) { @@ -1627,8 +1622,8 @@ static UniValue listsinceblock(const JSONRPCRequest& request) --*altheight; } - int last_height = tip_height ? *tip_height + 1 - target_confirms : -1; - uint256 lastblock = last_height >= 0 ? locked_chain->getBlockHash(last_height) : uint256(); + uint256 lastblock; + CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(pwallet->GetLastBlockHash(), pwallet->GetLastBlockHeight() + 1 - target_confirms, FoundBlock().hash(lastblock))); UniValue ret(UniValue::VOBJ); ret.pushKV("transactions", transactions); @@ -1912,44 +1907,52 @@ static UniValue walletpassphrase(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); - - if (!pwallet->IsCrypted()) { - throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletpassphrase was called."); - } + int64_t nSleepTime; + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); - // Note that the walletpassphrase is stored in request.params[0] which is not mlock()ed - SecureString strWalletPass; - strWalletPass.reserve(100); - // TODO: get rid of this .c_str() by implementing SecureString::operator=(std::string) - // Alternately, find a way to make request.params[0] mlock()'d to begin with. - strWalletPass = request.params[0].get_str().c_str(); + if (!pwallet->IsCrypted()) { + throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletpassphrase was called."); + } - // Get the timeout - int64_t nSleepTime = request.params[1].get_int64(); - // Timeout cannot be negative, otherwise it will relock immediately - if (nSleepTime < 0) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Timeout cannot be negative."); - } - // Clamp timeout - constexpr int64_t MAX_SLEEP_TIME = 100000000; // larger values trigger a macos/libevent bug? - if (nSleepTime > MAX_SLEEP_TIME) { - nSleepTime = MAX_SLEEP_TIME; - } + // Note that the walletpassphrase is stored in request.params[0] which is not mlock()ed + SecureString strWalletPass; + strWalletPass.reserve(100); + // TODO: get rid of this .c_str() by implementing SecureString::operator=(std::string) + // Alternately, find a way to make request.params[0] mlock()'d to begin with. + strWalletPass = request.params[0].get_str().c_str(); + + // Get the timeout + nSleepTime = request.params[1].get_int64(); + // Timeout cannot be negative, otherwise it will relock immediately + if (nSleepTime < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Timeout cannot be negative."); + } + // Clamp timeout + constexpr int64_t MAX_SLEEP_TIME = 100000000; // larger values trigger a macos/libevent bug? + if (nSleepTime > MAX_SLEEP_TIME) { + nSleepTime = MAX_SLEEP_TIME; + } - if (strWalletPass.empty()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "passphrase can not be empty"); - } + if (strWalletPass.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "passphrase can not be empty"); + } - if (!pwallet->Unlock(strWalletPass)) { - throw JSONRPCError(RPC_WALLET_PASSPHRASE_INCORRECT, "Error: The wallet passphrase entered was incorrect."); - } + if (!pwallet->Unlock(strWalletPass)) { + throw JSONRPCError(RPC_WALLET_PASSPHRASE_INCORRECT, "Error: The wallet passphrase entered was incorrect."); + } - pwallet->TopUpKeyPool(); + pwallet->TopUpKeyPool(); - pwallet->nRelockTime = GetTime() + nSleepTime; + pwallet->nRelockTime = GetTime() + nSleepTime; + } + // rpcRunLater must be called without cs_wallet held otherwise a deadlock + // can occur. The deadlock would happen when RPCRunLater removes the + // previous timer (and waits for the callback to finish if already running) + // and the callback locks cs_wallet. + AssertLockNotHeld(wallet->cs_wallet); // Keep a weak pointer to the wallet so that it is possible to unload the // wallet before the following callback is called. If a valid shared pointer // is acquired in the callback then the wallet is still loaded. @@ -2312,7 +2315,8 @@ static UniValue settxfee(const JSONRPCRequest& request) } RPCHelpMan{"settxfee", - "\nSet the transaction fee per kB for this wallet. Overrides the global -paytxfee command line parameter.\n", + "\nSet the transaction fee per kB for this wallet. Overrides the global -paytxfee command line parameter.\n" + "Can be deactivated by passing 0 as the fee. In that case automatic fee selection will be used by default.\n", { {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The transaction fee in " + CURRENCY_UNIT + "/kB"}, }, @@ -2330,12 +2334,15 @@ static UniValue settxfee(const JSONRPCRequest& request) CAmount nAmount = AmountFromValue(request.params[0]); CFeeRate tx_fee_rate(nAmount, 1000); + CFeeRate max_tx_fee_rate(pwallet->m_default_max_tx_fee, 1000); if (tx_fee_rate == CFeeRate(0)) { // automatic selection } else if (tx_fee_rate < pwallet->chain().relayMinFee()) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than min relay tx fee (%s)", pwallet->chain().relayMinFee().ToString())); } else if (tx_fee_rate < pwallet->m_min_fee) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than wallet min fee (%s)", pwallet->m_min_fee.ToString())); + } else if (tx_fee_rate > max_tx_fee_rate) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be more than wallet max tx fee (%s)", max_tx_fee_rate.ToString())); } pwallet->m_pay_tx_fee = tx_fee_rate; @@ -2574,7 +2581,7 @@ static UniValue loadwallet(const JSONRPCRequest& request) RPCHelpMan{"loadwallet", "\nLoads a wallet from a wallet file or directory." "\nNote that all wallet command-line options used when starting bitcoind will be" - "\napplied to the new wallet (eg -zapwallettxes, upgradewallet, rescan, etc).\n", + "\napplied to the new wallet (eg -zapwallettxes, rescan, etc).\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet directory or .dat file."}, }, @@ -2949,9 +2956,9 @@ static UniValue listunspent(const JSONRPCRequest& request) if (fValidAddress) { entry.pushKV("address", EncodeDestination(address)); - auto i = pwallet->mapAddressBook.find(address); - if (i != pwallet->mapAddressBook.end()) { - entry.pushKV("label", i->second.name); + const auto* address_book_entry = pwallet->FindAddressBookEntry(address); + if (address_book_entry) { + entry.pushKV("label", address_book_entry->GetLabel()); } std::unique_ptr<SigningProvider> provider = pwallet->GetSolvingProvider(scriptPubKey); @@ -3523,28 +3530,29 @@ UniValue rescanblockchain(const JSONRPCRequest& request) }, }.Check(request); - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } int start_height = 0; - uint256 start_block, stop_block; + Optional<int> stop_height; + uint256 start_block; { auto locked_chain = pwallet->chain().lock(); - Optional<int> tip_height = locked_chain->getHeight(); + LOCK(pwallet->cs_wallet); + int tip_height = pwallet->GetLastBlockHeight(); if (!request.params[0].isNull()) { start_height = request.params[0].get_int(); - if (start_height < 0 || !tip_height || start_height > *tip_height) { + if (start_height < 0 || start_height > tip_height) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid start_height"); } } - Optional<int> stop_height; if (!request.params[1].isNull()) { stop_height = request.params[1].get_int(); - if (*stop_height < 0 || !tip_height || *stop_height > *tip_height) { + if (*stop_height < 0 || *stop_height > tip_height) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stop_height"); } else if (*stop_height < start_height) { @@ -3553,25 +3561,15 @@ UniValue rescanblockchain(const JSONRPCRequest& request) } // We can't rescan beyond non-pruned blocks, stop and throw an error - if (locked_chain->findPruned(start_height, stop_height)) { + if (!pwallet->chain().hasBlocks(pwallet->GetLastBlockHash(), start_height, stop_height)) { throw JSONRPCError(RPC_MISC_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height."); } - if (tip_height) { - start_block = locked_chain->getBlockHash(start_height); - // If called with a stop_height, set the stop_height here to - // trigger a rescan to that height. - // If called without a stop height, leave stop_height as null here - // so rescan continues to the tip (even if the tip advances during - // rescan). - if (stop_height) { - stop_block = locked_chain->getBlockHash(*stop_height); - } - } + CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(pwallet->GetLastBlockHash(), start_height, FoundBlock().hash(start_block))); } CWallet::ScanResult result = - pwallet->ScanForWalletTransactions(start_block, stop_block, reserver, true /* fUpdate */); + pwallet->ScanForWalletTransactions(start_block, start_height, stop_height, reserver, true /* fUpdate */); switch (result.status) { case CWallet::ScanResult::SUCCESS: break; @@ -3698,7 +3696,7 @@ static UniValue AddressBookDataToJSON(const CAddressBookData& data, const bool v { UniValue ret(UniValue::VOBJ); if (verbose) { - ret.pushKV("name", data.name); + ret.pushKV("name", data.GetLabel()); } ret.pushKV("purpose", data.purpose); return ret; @@ -3808,8 +3806,9 @@ UniValue getaddressinfo(const JSONRPCRequest& request) // DEPRECATED: Return label field if existing. Currently only one label can // be associated with an address, so the label should be equivalent to the // value of the name key/value pair in the labels array below. - if ((pwallet->chain().rpcEnableDeprecated("label")) && (pwallet->mapAddressBook.count(dest))) { - ret.pushKV("label", pwallet->mapAddressBook.at(dest).name); + const auto* address_book_entry = pwallet->FindAddressBookEntry(dest); + if (pwallet->chain().rpcEnableDeprecated("label") && address_book_entry) { + ret.pushKV("label", address_book_entry->GetLabel()); } ret.pushKV("ischange", pwallet->IsChange(scriptPubKey)); @@ -3832,14 +3831,13 @@ UniValue getaddressinfo(const JSONRPCRequest& request) // stable if we allow multiple labels to be associated with an address in // the future. UniValue labels(UniValue::VARR); - std::map<CTxDestination, CAddressBookData>::const_iterator mi = pwallet->mapAddressBook.find(dest); - if (mi != pwallet->mapAddressBook.end()) { + if (address_book_entry) { // DEPRECATED: The previous behavior of returning an array containing a // JSON object of `name` and `purpose` key/value pairs is deprecated. if (pwallet->chain().rpcEnableDeprecated("labelspurpose")) { - labels.push_back(AddressBookDataToJSON(mi->second, true)); + labels.push_back(AddressBookDataToJSON(*address_book_entry, true)); } else { - labels.push_back(mi->second.name); + labels.push_back(address_book_entry->GetLabel()); } } ret.pushKV("labels", std::move(labels)); @@ -3883,10 +3881,11 @@ static UniValue getaddressesbylabel(const JSONRPCRequest& request) // Find all addresses that have the given label UniValue ret(UniValue::VOBJ); std::set<std::string> addresses; - for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { - if (item.second.name == label) { + for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->m_address_book) { + if (item.second.IsChange()) continue; + if (item.second.GetLabel() == label) { std::string address = EncodeDestination(item.first); - // CWallet::mapAddressBook is not expected to contain duplicate + // CWallet::m_address_book is not expected to contain duplicate // address strings, but build a separate set as a precaution just in // case it does. bool unique = addresses.emplace(address).second; @@ -3947,9 +3946,10 @@ static UniValue listlabels(const JSONRPCRequest& request) // Add to a set to sort by label name, then insert into Univalue array std::set<std::string> label_set; - for (const std::pair<const CTxDestination, CAddressBookData>& entry : pwallet->mapAddressBook) { + for (const std::pair<const CTxDestination, CAddressBookData>& entry : pwallet->m_address_book) { + if (entry.second.IsChange()) continue; if (purpose.empty() || entry.second.purpose == purpose) { - label_set.insert(entry.second.name); + label_set.insert(entry.second.GetLabel()); } } @@ -4007,7 +4007,7 @@ UniValue sethdseed(const JSONRPCRequest& request) // Do not do anything to non-HD wallets if (!pwallet->CanSupportFeature(FEATURE_HD)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Start with -upgradewallet in order to upgrade a non-HD wallet to HD"); + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Use the upgradewallet RPC in order to upgrade a non-HD wallet to HD"); } EnsureWalletIsUnlocked(pwallet); @@ -4231,6 +4231,45 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) return result; } +static UniValue upgradewallet(const JSONRPCRequest& request) +{ + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + CWallet* const pwallet = wallet.get(); + + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + RPCHelpMan{"upgradewallet", + "\nUpgrade the wallet. Upgrades to the latest version if no version number is specified\n" + "New keys may be generated and a new wallet backup will need to be made.", + { + {"version", RPCArg::Type::NUM, /* default */ strprintf("%d", FEATURE_LATEST), "The version number to upgrade to. Default is the latest wallet version"} + }, + RPCResults{}, + RPCExamples{ + HelpExampleCli("upgradewallet", "169900") + + HelpExampleRpc("upgradewallet", "169900") + } + }.Check(request); + + RPCTypeCheck(request.params, {UniValue::VNUM}, true); + + EnsureWalletIsUnlocked(pwallet); + + int version = 0; + if (!request.params[0].isNull()) { + version = request.params[0].get_int(); + } + + std::string error; + std::vector<std::string> warnings; + if (!pwallet->UpgradeWallet(version, error, warnings)) { + throw JSONRPCError(RPC_WALLET_ERROR, error); + } + return error; +} + UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp UniValue importprivkey(const JSONRPCRequest& request); @@ -4242,6 +4281,8 @@ UniValue importprunedfunds(const JSONRPCRequest& request); UniValue removeprunedfunds(const JSONRPCRequest& request); UniValue importmulti(const JSONRPCRequest& request); +void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers) +{ // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -4297,6 +4338,7 @@ static const CRPCCommand commands[] = { "wallet", "signmessage", &signmessage, {"address","message"} }, { "wallet", "signrawtransactionwithwallet", &signrawtransactionwithwallet, {"hexstring","prevtxs","sighashtype"} }, { "wallet", "unloadwallet", &unloadwallet, {"wallet_name"} }, + { "wallet", "upgradewallet", &upgradewallet, {"version"} }, { "wallet", "walletcreatefundedpsbt", &walletcreatefundedpsbt, {"inputs","outputs","locktime","options","bip32derivs"} }, { "wallet", "walletlock", &walletlock, {} }, { "wallet", "walletpassphrase", &walletpassphrase, {"passphrase","timeout"} }, @@ -4305,8 +4347,6 @@ static const CRPCCommand commands[] = }; // clang-format on -void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers) -{ for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) handlers.emplace_back(chain.handleRpc(commands[vcidx])); } diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index a7a29de9c6..8c149d455b 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index b96cb0aa1a..ec75f06e5e 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,7 +20,7 @@ bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestinat // Generate a new key that is added to wallet CPubKey new_key; if (!GetKeyFromPool(new_key, type)) { - error = "Error: Keypool ran out, please call keypoolrefill first"; + error = _("Error: Keypool ran out, please call keypoolrefill first").translated; return false; } LearnRelatedScripts(new_key, type); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 21d57cb898..66f4542cf9 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -1,16 +1,16 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <node/context.h> -#include <wallet/wallet.h> -#include <wallet/coinselection.h> -#include <wallet/coincontrol.h> #include <amount.h> +#include <node/context.h> #include <primitives/transaction.h> #include <random.h> #include <test/util/setup_common.h> +#include <wallet/coincontrol.h> +#include <wallet/coinselection.h> #include <wallet/test/wallet_test_fixture.h> +#include <wallet/wallet.h> #include <boost/test/unit_test.hpp> #include <random> @@ -176,8 +176,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) selection.clear(); // Select 5 Cent - add_coin(3 * CENT, 3, actual_selection); - add_coin(2 * CENT, 2, actual_selection); + add_coin(4 * CENT, 4, actual_selection); + add_coin(1 * CENT, 1, actual_selection); BOOST_CHECK(SelectCoinsBnB(GroupCoins(utxo_pool), 5 * CENT, 0.5 * CENT, selection, value_ret, not_input_fees)); BOOST_CHECK(equal_sets(selection, actual_selection)); BOOST_CHECK_EQUAL(value_ret, 5 * CENT); @@ -204,9 +204,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) // Select 10 Cent add_coin(5 * CENT, 5, utxo_pool); + add_coin(5 * CENT, 5, actual_selection); add_coin(4 * CENT, 4, actual_selection); - add_coin(3 * CENT, 3, actual_selection); - add_coin(2 * CENT, 2, actual_selection); add_coin(1 * CENT, 1, actual_selection); BOOST_CHECK(SelectCoinsBnB(GroupCoins(utxo_pool), 10 * CENT, 0.5 * CENT, selection, value_ret, not_input_fees)); BOOST_CHECK(equal_sets(selection, actual_selection)); diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 8b7b7af21d..35860577cd 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index b9e714946d..7ba3148ff3 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index 81d8a60b8a..a294935b64 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index a487e9e2e0..9e1c1ce0be 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -46,7 +46,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); - // Verify ScanForWalletTransactions accommodates a null start block. + // Verify ScanForWalletTransactions fails to read an unknown start block. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); { @@ -54,10 +54,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions({} /* start_block */, {} /* stop_block */, reserver, false /* update */); - BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); + CWallet::ScanResult result = wallet.ScanForWalletTransactions({} /* start_block */, 0 /* start_height */, {} /* max_height */, reserver, false /* update */); + BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK(result.last_scanned_block.IsNull()); BOOST_CHECK(!result.last_scanned_height); @@ -73,9 +73,9 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -96,9 +96,9 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, oldTip->GetBlockHash()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -118,9 +118,9 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, newTip->GetBlockHash()); BOOST_CHECK(result.last_scanned_block.IsNull()); @@ -152,6 +152,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); wallet->SetupLegacyScriptPubKeyMan(); + WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); AddWallet(wallet); UniValue keys; keys.setArray(); @@ -225,6 +226,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) request.params.setArray(); request.params.push_back(backup_file); AddWallet(wallet); + wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); ::dumpwallet(request); RemoveWallet(wallet); } @@ -233,16 +235,17 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // were scanned, and no prior blocks were scanned. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); JSONRPCRequest request; request.params.setArray(); request.params.push_back(backup_file); AddWallet(wallet); + wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); ::importwallet(request); RemoveWallet(wallet); - LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.size(), 3U); BOOST_CHECK_EQUAL(m_coinbase_txns.size(), 103U); for (size_t i = 0; i < m_coinbase_txns.size(); ++i) { @@ -454,15 +457,15 @@ public: CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); { - LOCK(wallet->cs_wallet); + LOCK2(::cs_main, wallet->cs_wallet); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); - WalletRescanReserver reserver(wallet.get()); + WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(::ChainActive().Genesis()->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(::ChainActive().Genesis()->GetBlockHash(), 0 /* start_height */, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK_EQUAL(result.last_scanned_block, ::ChainActive().Tip()->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, ::ChainActive().Height()); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index bc9f84a11d..7a6deab1a8 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -22,6 +22,7 @@ #include <script/script.h> #include <script/signingprovider.h> #include <util/bip32.h> +#include <util/check.h> #include <util/error.h> #include <util/fees.h> #include <util/moneystr.h> @@ -35,6 +36,8 @@ #include <boost/algorithm/string/replace.hpp> +using interfaces::FoundBlock; + const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS{ {WALLET_FLAG_AVOID_REUSE, "You need to rescan the blockchain in order to correctly mark used " @@ -1242,8 +1245,9 @@ bool CWallet::IsChange(const CScript& script) const return true; LOCK(cs_wallet); - if (!mapAddressBook.count(address)) + if (!FindAddressBookEntry(address)) { return true; + } } return false; } @@ -1593,22 +1597,17 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r // Find starting block. May be null if nCreateTime is greater than the // highest blockchain timestamp, in which case there is nothing that needs // to be scanned. + int start_height = 0; uint256 start_block; - { - auto locked_chain = chain().lock(); - const Optional<int> start_height = locked_chain->findFirstBlockWithTimeAndHeight(startTime - TIMESTAMP_WINDOW, 0, &start_block); - const Optional<int> tip_height = locked_chain->getHeight(); - WalletLogPrintf("%s: Rescanning last %i blocks\n", __func__, tip_height && start_height ? *tip_height - *start_height + 1 : 0); - } + bool start = chain().findFirstBlockWithTimeAndHeight(startTime - TIMESTAMP_WINDOW, 0, FoundBlock().hash(start_block).height(start_height)); + WalletLogPrintf("%s: Rescanning last %i blocks\n", __func__, start ? WITH_LOCK(cs_wallet, return GetLastBlockHeight()) - start_height + 1 : 0); - if (!start_block.IsNull()) { + if (start) { // TODO: this should take into account failure by ScanResult::USER_ABORT - ScanResult result = ScanForWalletTransactions(start_block, {} /* stop_block */, reserver, update); + ScanResult result = ScanForWalletTransactions(start_block, start_height, {} /* max_height */, reserver, update); if (result.status == ScanResult::FAILURE) { int64_t time_max; - if (!chain().findBlock(result.last_failed_block, nullptr /* block */, nullptr /* time */, &time_max)) { - throw std::logic_error("ScanForWalletTransactions returned invalid block hash"); - } + CHECK_NONFATAL(chain().findBlock(result.last_failed_block, FoundBlock().maxTime(time_max))); return time_max + TIMESTAMP_WINDOW + 1; } } @@ -1622,9 +1621,9 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r * * @param[in] start_block Scan starting block. If block is not on the active * chain, the scan will return SUCCESS immediately. - * @param[in] stop_block Scan ending block. If block is not on the active - * chain, the scan will continue until it reaches the - * chain tip. + * @param[in] start_height Height of start_block + * @param[in] max_height Optional max scanning height. If unset there is + * no maximum and scanning can continue to the tip * * @return ScanResult returning scan information and indicating success or * failure. Return status will be set to SUCCESS if scan was @@ -1636,7 +1635,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r * the main chain after to the addition of any new keys you want to detect * transactions for. */ -CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, const uint256& stop_block, const WalletRescanReserver& reserver, bool fUpdate) +CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, Optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate) { int64_t nNow = GetTime(); int64_t start_time = GetTimeMillis(); @@ -1650,36 +1649,32 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc fAbortRescan = false; ShowProgress(strprintf("%s " + _("Rescanning...").translated, GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup - uint256 tip_hash; - // The way the 'block_height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. - Optional<int> block_height = MakeOptional(false, int()); - double progress_begin; - double progress_end; - { - auto locked_chain = chain().lock(); - if (Optional<int> tip_height = locked_chain->getHeight()) { - tip_hash = locked_chain->getBlockHash(*tip_height); - } - block_height = locked_chain->getBlockHeight(block_hash); - progress_begin = chain().guessVerificationProgress(block_hash); - progress_end = chain().guessVerificationProgress(stop_block.IsNull() ? tip_hash : stop_block); - } + uint256 tip_hash = WITH_LOCK(cs_wallet, return GetLastBlockHash()); + uint256 end_hash = tip_hash; + if (max_height) chain().findAncestorByHeight(tip_hash, *max_height, FoundBlock().hash(end_hash)); + double progress_begin = chain().guessVerificationProgress(block_hash); + double progress_end = chain().guessVerificationProgress(end_hash); double progress_current = progress_begin; - while (block_height && !fAbortRescan && !chain().shutdownRequested()) { + int block_height = start_height; + while (!fAbortRescan && !chain().shutdownRequested()) { m_scanning_progress = (progress_current - progress_begin) / (progress_end - progress_begin); - if (*block_height % 100 == 0 && progress_end - progress_begin > 0.0) { + if (block_height % 100 == 0 && progress_end - progress_begin > 0.0) { ShowProgress(strprintf("%s " + _("Rescanning...").translated, GetDisplayName()), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); } if (GetTime() >= nNow + 60) { nNow = GetTime(); - WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", *block_height, progress_current); + WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } CBlock block; - if (chain().findBlock(block_hash, &block) && !block.IsNull()) { + bool next_block; + uint256 next_block_hash; + bool reorg = false; + if (chain().findBlock(block_hash, FoundBlock().data(block)) && !block.IsNull()) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - if (!locked_chain->getBlockHeight(block_hash)) { + next_block = chain().findNextBlock(block_hash, block_height, FoundBlock().hash(next_block_hash), &reorg); + if (reorg) { // Abort scan if current block is no longer active, to prevent // marking transactions as coming from the wrong block. // TODO: This should return success instead of failure, see @@ -1689,37 +1684,38 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *block_height, block_hash, posInBlock); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, block_height, block_hash, posInBlock); SyncTransaction(block.vtx[posInBlock], confirm, fUpdate); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; - result.last_scanned_height = *block_height; + result.last_scanned_height = block_height; } else { // could not scan block, keep scanning but record this block as the most recent failure result.last_failed_block = block_hash; result.status = ScanResult::FAILURE; + next_block = chain().findNextBlock(block_hash, block_height, FoundBlock().hash(next_block_hash), &reorg); } - if (block_hash == stop_block) { + if (max_height && block_height >= *max_height) { break; } { auto locked_chain = chain().lock(); - Optional<int> tip_height = locked_chain->getHeight(); - if (!tip_height || *tip_height <= block_height || !locked_chain->getBlockHeight(block_hash)) { + if (!next_block || reorg) { // break successfully when rescan has reached the tip, or // previous block is no longer on the chain due to a reorg break; } // increment block and verification progress - block_hash = locked_chain->getBlockHash(++*block_height); + block_hash = next_block_hash; + ++block_height; progress_current = chain().guessVerificationProgress(block_hash); // handle updated tip hash const uint256 prev_tip_hash = tip_hash; - tip_hash = locked_chain->getBlockHash(*tip_height); - if (stop_block.IsNull() && prev_tip_hash != tip_hash) { + tip_hash = WITH_LOCK(cs_wallet, return GetLastBlockHash()); + if (!max_height && prev_tip_hash != tip_hash) { // in case the tip has changed, update progress max progress_end = chain().guessVerificationProgress(tip_hash); } @@ -1727,10 +1723,10 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc } ShowProgress(strprintf("%s " + _("Rescanning...").translated, GetDisplayName()), 100); // hide progress dialog in GUI if (block_height && fAbortRescan) { - WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", *block_height, progress_current); + WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; } else if (block_height && chain().shutdownRequested()) { - WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", *block_height, progress_current); + WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; } else { WalletLogPrintf("Rescan completed in %15dms\n", GetTimeMillis() - start_time); @@ -2376,6 +2372,13 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm ++it; } + unsigned int limit_ancestor_count = 0; + unsigned int limit_descendant_count = 0; + chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); + size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); + size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); + bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); + // form groups from remaining coins; note that preset coins will not // automatically have their associated (same address) coins included if (coin_control.m_avoid_partial_spends && vCoins.size() > OUTPUT_GROUP_MAX_ENTRIES) { @@ -2384,14 +2387,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // explicitly shuffling the outputs before processing Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); } - std::vector<OutputGroup> groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends); - - unsigned int limit_ancestor_count; - unsigned int limit_descendant_count; - chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); - size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); - size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); - bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); + std::vector<OutputGroup> groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends, max_ancestors); bool res = value_to_select <= 0 || SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || @@ -2630,13 +2626,15 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC return true; } -static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain) +static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, const uint256& block_hash) { if (chain.isInitialBlockDownload()) { return false; } constexpr int64_t MAX_ANTI_FEE_SNIPING_TIP_AGE = 8 * 60 * 60; // in seconds - if (locked_chain.getBlockTime(*locked_chain.getHeight()) < (GetTime() - MAX_ANTI_FEE_SNIPING_TIP_AGE)) { + int64_t block_time; + CHECK_NONFATAL(chain.findBlock(block_hash, FoundBlock().time(block_time))); + if (block_time < (GetTime() - MAX_ANTI_FEE_SNIPING_TIP_AGE)) { return false; } return true; @@ -2646,9 +2644,8 @@ static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, interfaces::Cha * Return a height-based locktime for new transactions (uses the height of the * current chain tip unless we are not synced with the current chain */ -static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain) +static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uint256& block_hash, int block_height) { - uint32_t const height = locked_chain.getHeight().get_value_or(-1); uint32_t locktime; // Discourage fee sniping. // @@ -2670,8 +2667,8 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interface // enough, that fee sniping isn't a problem yet, but by implementing a fix // now we ensure code won't be written that makes assumptions about // nLockTime that preclude a fix later. - if (IsCurrentForAntiFeeSniping(chain, locked_chain)) { - locktime = height; + if (IsCurrentForAntiFeeSniping(chain, block_hash)) { + locktime = block_height; // Secondly occasionally randomly pick a nLockTime even further back, so // that transactions that are delayed after signing for whatever reason, @@ -2685,7 +2682,6 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interface // unique "nLockTime fingerprint", set nLockTime to a constant. locktime = 0; } - assert(locktime <= height); assert(locktime < LOCKTIME_THRESHOLD); return locktime; } @@ -2745,9 +2741,6 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } CMutableTransaction txNew; - - txNew.nLockTime = GetLocktimeForNewTransaction(chain(), locked_chain); - FeeCalculation feeCalc; CAmount nFeeNeeded; int nBytes; @@ -2755,6 +2748,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std std::set<CInputCoin> setCoins; auto locked_chain = chain().lock(); LOCK(cs_wallet); + txNew.nLockTime = GetLocktimeForNewTransaction(chain(), GetLastBlockHash(), GetLastBlockHeight()); { std::vector<COutput> vAvailableCoins; AvailableCoins(*locked_chain, vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); @@ -2776,20 +2770,14 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // rediscover unknown transactions that were written with keys of ours to recover // post-backup change. - // Reserve a new key pair from key pool - if (!CanGetAddresses(true)) { - strFailReason = _("Can't generate a change-address key. No keys in the internal keypool and can't generate any keys.").translated; - return false; - } + // Reserve a new key pair from key pool. If it fails, provide a dummy + // destination in case we don't need change. CTxDestination dest; - bool ret = reservedest.GetReservedDestination(dest, true); - if (!ret) - { - strFailReason = "Keypool ran out, please call keypoolrefill first"; - return false; + if (!reservedest.GetReservedDestination(dest, true)) { + strFailReason = _("Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.").translated; } - scriptChange = GetScriptForDestination(dest); + assert(!dest.empty() || scriptChange.empty()); } CTxOut change_prototype_txout(0, scriptChange); coin_selection_params.change_output_size = GetSerializeSize(change_prototype_txout); @@ -3005,6 +2993,11 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std coin_selection_params.use_bnb = false; continue; } + + // Give up if change keypool ran out and we failed to find a solution without change: + if (scriptChange.empty() && nChangePosInOut != -1) { + return false; + } } // Shuffle selected coins and fill in final vin @@ -3196,11 +3189,11 @@ bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& add bool fUpdated = false; { LOCK(cs_wallet); - std::map<CTxDestination, CAddressBookData>::iterator mi = mapAddressBook.find(address); - fUpdated = mi != mapAddressBook.end(); - mapAddressBook[address].name = strName; + std::map<CTxDestination, CAddressBookData>::iterator mi = m_address_book.find(address); + fUpdated = (mi != m_address_book.end() && !mi->second.IsChange()); + m_address_book[address].SetLabel(strName); if (!strPurpose.empty()) /* update purpose only if requested */ - mapAddressBook[address].purpose = strPurpose; + m_address_book[address].purpose = strPurpose; } NotifyAddressBookChanged(this, address, strName, IsMine(address) != ISMINE_NO, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); @@ -3217,16 +3210,24 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s bool CWallet::DelAddressBook(const CTxDestination& address) { + // If we want to delete receiving addresses, we need to take care that DestData "used" (and possibly newer DestData) gets preserved (and the "deleted" address transformed into a change entry instead of actually being deleted) + // NOTE: This isn't a problem for sending addresses because they never have any DestData yet! + // When adding new DestData, it should be considered here whether to retain or delete it (or move it?). + if (IsMine(address)) { + WalletLogPrintf("%s called with IsMine address, NOT SUPPORTED. Please report this bug! %s\n", __func__, PACKAGE_BUGREPORT); + return false; + } + { LOCK(cs_wallet); // Delete destdata tuples associated with address std::string strAddress = EncodeDestination(address); - for (const std::pair<const std::string, std::string> &item : mapAddressBook[address].destdata) + for (const std::pair<const std::string, std::string> &item : m_address_book[address].destdata) { WalletBatch(*database).EraseDestData(strAddress, item.first); } - mapAddressBook.erase(address); + m_address_book.erase(address); } NotifyAddressBookChanged(this, address, "", IsMine(address) != ISMINE_NO, "", CT_DELETED); @@ -3292,7 +3293,7 @@ bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& des ReserveDestination reservedest(this, type); if (!reservedest.GetReservedDestination(dest, true)) { - error = "Error: Keypool ran out, please call keypoolrefill first"; + error = _("Error: Keypool ran out, please call keypoolrefill first").translated; return false; } @@ -3462,10 +3463,11 @@ std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) co { LOCK(cs_wallet); std::set<CTxDestination> result; - for (const std::pair<const CTxDestination, CAddressBookData>& item : mapAddressBook) + for (const std::pair<const CTxDestination, CAddressBookData>& item : m_address_book) { + if (item.second.IsChange()) continue; const CTxDestination& address = item.first; - const std::string& strName = item.second.name; + const std::string& strName = item.second.GetLabel(); if (strName == label) result.insert(address); } @@ -3566,12 +3568,13 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C } // map in which we'll infer heights of other keys - const Optional<int> tip_height = locked_chain.getHeight(); - const int max_height = tip_height && *tip_height > 144 ? *tip_height - 144 : 0; // the tip can be reorganized; use a 144-block safety margin - std::map<CKeyID, int> mapKeyFirstBlock; + std::map<CKeyID, const CWalletTx::Confirmation*> mapKeyFirstBlock; + CWalletTx::Confirmation max_confirm; + max_confirm.block_height = GetLastBlockHeight() > 144 ? GetLastBlockHeight() - 144 : 0; // the tip can be reorganized; use a 144-block safety margin + CHECK_NONFATAL(chain().findAncestorByHeight(GetLastBlockHash(), max_confirm.block_height, FoundBlock().hash(max_confirm.hashBlock))); for (const CKeyID &keyid : spk_man->GetKeys()) { if (mapKeyBirth.count(keyid) == 0) - mapKeyFirstBlock[keyid] = max_height; + mapKeyFirstBlock[keyid] = &max_confirm; } // if there are no such keys, we're done @@ -3582,23 +3585,27 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C for (const auto& entry : mapWallet) { // iterate over all wallet transactions... const CWalletTx &wtx = entry.second; - if (Optional<int> height = locked_chain.getBlockHeight(wtx.m_confirm.hashBlock)) { + if (wtx.m_confirm.status == CWalletTx::CONFIRMED) { // ... which are already in a block for (const CTxOut &txout : wtx.tx->vout) { // iterate over all their outputs for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *spk_man)) { // ... and all their affected keys - std::map<CKeyID, int>::iterator rit = mapKeyFirstBlock.find(keyid); - if (rit != mapKeyFirstBlock.end() && *height < rit->second) - rit->second = *height; + auto rit = mapKeyFirstBlock.find(keyid); + if (rit != mapKeyFirstBlock.end() && wtx.m_confirm.block_height < rit->second->block_height) { + rit->second = &wtx.m_confirm; + } } } } } // Extract block timestamps for those keys - for (const auto& entry : mapKeyFirstBlock) - mapKeyBirth[entry.first] = locked_chain.getBlockTime(entry.second) - TIMESTAMP_WINDOW; // block times can be 2h off + for (const auto& entry : mapKeyFirstBlock) { + int64_t block_time; + CHECK_NONFATAL(chain().findBlock(entry.second->hashBlock, FoundBlock().time(block_time))); + mapKeyBirth[entry.first] = block_time - TIMESTAMP_WINDOW; // block times can be 2h off + } } /** @@ -3627,7 +3634,7 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const unsigned int nTimeSmart = wtx.nTimeReceived; if (!wtx.isUnconfirmed() && !wtx.isAbandoned()) { int64_t blocktime; - if (chain().findBlock(wtx.m_confirm.hashBlock, nullptr /* block */, &blocktime)) { + if (chain().findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(blocktime))) { int64_t latestNow = wtx.nTimeReceived; int64_t latestEntry = 0; @@ -3666,26 +3673,26 @@ bool CWallet::AddDestData(WalletBatch& batch, const CTxDestination &dest, const if (boost::get<CNoDestination>(&dest)) return false; - mapAddressBook[dest].destdata.insert(std::make_pair(key, value)); + m_address_book[dest].destdata.insert(std::make_pair(key, value)); return batch.WriteDestData(EncodeDestination(dest), key, value); } bool CWallet::EraseDestData(WalletBatch& batch, const CTxDestination &dest, const std::string &key) { - if (!mapAddressBook[dest].destdata.erase(key)) + if (!m_address_book[dest].destdata.erase(key)) return false; return batch.EraseDestData(EncodeDestination(dest), key); } void CWallet::LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value) { - mapAddressBook[dest].destdata.insert(std::make_pair(key, value)); + m_address_book[dest].destdata.insert(std::make_pair(key, value)); } bool CWallet::GetDestData(const CTxDestination &dest, const std::string &key, std::string *value) const { - std::map<CTxDestination, CAddressBookData>::const_iterator i = mapAddressBook.find(dest); - if(i != mapAddressBook.end()) + std::map<CTxDestination, CAddressBookData>::const_iterator i = m_address_book.find(dest); + if(i != m_address_book.end()) { CAddressBookData::StringMap::const_iterator j = i->second.destdata.find(key); if(j != i->second.destdata.end()) @@ -3701,7 +3708,7 @@ bool CWallet::GetDestData(const CTxDestination &dest, const std::string &key, st std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const { std::vector<std::string> values; - for (const auto& address : mapAddressBook) { + for (const auto& address : m_address_book) { for (const auto& data : address.second.destdata) { if (!data.first.compare(0, prefix.size(), prefix)) { values.emplace_back(data.second); @@ -3820,44 +3827,6 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - int prev_version = walletInstance->GetVersion(); - if (gArgs.GetBoolArg("-upgradewallet", fFirstRun)) - { - int nMaxVersion = gArgs.GetArg("-upgradewallet", 0); - if (nMaxVersion == 0) // the -upgradewallet without argument case - { - walletInstance->WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); - nMaxVersion = FEATURE_LATEST; - walletInstance->SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately - } - else - walletInstance->WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); - if (nMaxVersion < walletInstance->GetVersion()) - { - error = _("Cannot downgrade wallet").translated; - return nullptr; - } - walletInstance->SetMaxVersion(nMaxVersion); - } - - // Upgrade to HD if explicit upgrade - if (gArgs.GetBoolArg("-upgradewallet", false)) { - LOCK(walletInstance->cs_wallet); - - // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT - int max_version = walletInstance->GetVersion(); - if (!walletInstance->CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { - error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use -upgradewallet=169900 or -upgradewallet with no version specified.").translated; - return nullptr; - } - - for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { - if (!spk_man->Upgrade(prev_version, error)) { - return nullptr; - } - } - } - if (fFirstRun) { // ensure this wallet.dat can only be opened by clients supporting HD with chain split and expects no default key @@ -4054,8 +4023,8 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } { - WalletRescanReserver reserver(walletInstance.get()); - if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), {} /* stop block */, reserver, true /* update */).status)) { + WalletRescanReserver reserver(*walletInstance); + if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { error = _("Failed to rescan the wallet during initialization").translated; return nullptr; } @@ -4103,12 +4072,58 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, { walletInstance->WalletLogPrintf("setKeyPool.size() = %u\n", walletInstance->GetKeyPoolSize()); walletInstance->WalletLogPrintf("mapWallet.size() = %u\n", walletInstance->mapWallet.size()); - walletInstance->WalletLogPrintf("mapAddressBook.size() = %u\n", walletInstance->mapAddressBook.size()); + walletInstance->WalletLogPrintf("m_address_book.size() = %u\n", walletInstance->m_address_book.size()); } return walletInstance; } +const CAddressBookData* CWallet::FindAddressBookEntry(const CTxDestination& dest, bool allow_change) const +{ + const auto& address_book_it = m_address_book.find(dest); + if (address_book_it == m_address_book.end()) return nullptr; + if ((!allow_change) && address_book_it->second.IsChange()) { + return nullptr; + } + return &address_book_it->second; +} + +bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings) +{ + int prev_version = GetVersion(); + int nMaxVersion = version; + if (nMaxVersion == 0) // the -upgradewallet without argument case + { + WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); + nMaxVersion = FEATURE_LATEST; + SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately + } + else + WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); + if (nMaxVersion < GetVersion()) + { + error = _("Cannot downgrade wallet").translated; + return false; + } + SetMaxVersion(nMaxVersion); + + LOCK(cs_wallet); + + // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT + int max_version = GetVersion(); + if (!CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { + error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified.").translated; + return false; + } + + for (auto spk_man : GetActiveScriptPubKeyMans()) { + if (!spk_man->Upgrade(prev_version, error)) { + return false; + } + } + return true; +} + void CWallet::postInitProcess() { auto locked_chain = chain().lock(); @@ -4166,32 +4181,49 @@ bool CWalletTx::IsImmatureCoinBase() const return GetBlocksToMaturity() > 0; } -std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const { +std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const { std::vector<OutputGroup> groups; std::map<CTxDestination, OutputGroup> gmap; - CTxDestination dst; + std::set<CTxDestination> full_groups; + for (const auto& output : outputs) { if (output.fSpendable) { + CTxDestination dst; CInputCoin input_coin = output.GetInputCoin(); size_t ancestors, descendants; chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); if (!single_coin && ExtractDestination(output.tx->tx->vout[output.i].scriptPubKey, dst)) { - // Limit output groups to no more than 10 entries, to protect - // against inadvertently creating a too-large transaction - // when using -avoidpartialspends - if (gmap[dst].m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { - groups.push_back(gmap[dst]); - gmap.erase(dst); + auto it = gmap.find(dst); + if (it != gmap.end()) { + // Limit output groups to no more than OUTPUT_GROUP_MAX_ENTRIES + // number of entries, to protect against inadvertently creating + // a too-large transaction when using -avoidpartialspends to + // prevent breaking consensus or surprising users with a very + // high amount of fees. + if (it->second.m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { + groups.push_back(it->second); + it->second = OutputGroup{}; + full_groups.insert(dst); + } + it->second.Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); + } else { + gmap[dst].Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); } - gmap[dst].Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); } else { groups.emplace_back(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); } } } if (!single_coin) { - for (const auto& it : gmap) groups.push_back(it.second); + for (auto& it : gmap) { + auto& group = it.second; + if (full_groups.count(it.first) > 0) { + // Make this unattractive as we want coin selection to avoid it if possible + group.m_ancestors = max_ancestors - 1; + } + groups.push_back(group); + } } return groups; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index fe59773488..176c483572 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -181,14 +181,23 @@ public: /** Address book data */ class CAddressBookData { +private: + bool m_change{true}; + std::string m_label; public: - std::string name; std::string purpose; CAddressBookData() : purpose("unknown") {} typedef std::map<std::string, std::string> StringMap; StringMap destdata; + + bool IsChange() const { return m_change; } + const std::string& GetLabel() const { return m_label; } + void SetLabel(const std::string& label) { + m_change = false; + m_label = label; + } }; struct CRecipient @@ -775,7 +784,8 @@ public: int64_t nOrderPosNext GUARDED_BY(cs_wallet) = 0; uint64_t nAccountingEntryNumber = 0; - std::map<CTxDestination, CAddressBookData> mapAddressBook GUARDED_BY(cs_wallet); + std::map<CTxDestination, CAddressBookData> m_address_book GUARDED_BY(cs_wallet); + const CAddressBookData* FindAddressBookEntry(const CTxDestination&, bool allow_change = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); std::set<COutPoint> setLockedCoins GUARDED_BY(cs_wallet); @@ -820,7 +830,7 @@ public: bool IsSpentKey(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetSpentKeyState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const; + std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const; bool IsLockedCoin(uint256 hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void LockCoin(const COutPoint& output) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -842,7 +852,10 @@ public: bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; nWalletMaxVersion = std::max(nWalletMaxVersion, nVersion); return true; } - //! Adds a destination data tuple to the store, and saves it to disk + /** + * Adds a destination data tuple to the store, and saves it to disk + * When adding new fields, take care to consider how DelAddressBook should handle it! + */ bool AddDestData(WalletBatch& batch, const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Erases a destination data tuple in the store and on disk bool EraseDestData(WalletBatch& batch, const CTxDestination& dest, const std::string& key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -894,7 +907,7 @@ public: //! USER_ABORT. uint256 last_failed_block; }; - ScanResult ScanForWalletTransactions(const uint256& first_block, const uint256& last_block, const WalletRescanReserver& reserver, bool fUpdate); + ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, Optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate); void transactionRemovedFromMempool(const CTransactionRef &ptx) override; void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); @@ -1162,6 +1175,9 @@ public: LogPrintf(("%s " + fmt).c_str(), GetDisplayName(), parameters...); }; + /** Upgrade the wallet */ + bool UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings); + //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; @@ -1200,6 +1216,12 @@ public: assert(m_last_block_processed_height >= 0); return m_last_block_processed_height; }; + uint256 GetLastBlockHash() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) + { + AssertLockHeld(cs_wallet); + assert(m_last_block_processed_height >= 0); + return m_last_block_processed; + } /** Set last block processed height, currently only use in unit test */ void SetLastBlockProcessed(int block_height, uint256 block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { @@ -1222,35 +1244,35 @@ void MaybeResendWalletTxs(); class WalletRescanReserver { private: - CWallet* m_wallet; + CWallet& m_wallet; bool m_could_reserve; public: - explicit WalletRescanReserver(CWallet* w) : m_wallet(w), m_could_reserve(false) {} + explicit WalletRescanReserver(CWallet& w) : m_wallet(w), m_could_reserve(false) {} bool reserve() { assert(!m_could_reserve); - std::lock_guard<std::mutex> lock(m_wallet->mutexScanning); - if (m_wallet->fScanningWallet) { + std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); + if (m_wallet.fScanningWallet) { return false; } - m_wallet->m_scanning_start = GetTimeMillis(); - m_wallet->m_scanning_progress = 0; - m_wallet->fScanningWallet = true; + m_wallet.m_scanning_start = GetTimeMillis(); + m_wallet.m_scanning_progress = 0; + m_wallet.fScanningWallet = true; m_could_reserve = true; return true; } bool isReserved() const { - return (m_could_reserve && m_wallet->fScanningWallet); + return (m_could_reserve && m_wallet.fScanningWallet); } ~WalletRescanReserver() { - std::lock_guard<std::mutex> lock(m_wallet->mutexScanning); + std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); if (m_could_reserve) { - m_wallet->fScanningWallet = false; + m_wallet.fScanningWallet = false; } } }; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index a1928f45c4..f457db0a56 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -206,11 +206,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, if (strType == DBKeys::NAME) { std::string strAddress; ssKey >> strAddress; - ssValue >> pwallet->mapAddressBook[DecodeDestination(strAddress)].name; + std::string label; + ssValue >> label; + pwallet->m_address_book[DecodeDestination(strAddress)].SetLabel(label); } else if (strType == DBKeys::PURPOSE) { std::string strAddress; ssKey >> strAddress; - ssValue >> pwallet->mapAddressBook[DecodeDestination(strAddress)].purpose; + ssValue >> pwallet->m_address_book[DecodeDestination(strAddress)].purpose; } else if (strType == DBKeys::TX) { uint256 hash; ssKey >> hash; diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 1a65125480..a436f60ab3 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index fbfdf9dd6b..65643669c2 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -99,7 +99,7 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "HD (hd seed available): %s\n", wallet_instance->IsHDEnabled() ? "yes" : "no"); tfm::format(std::cout, "Keypool Size: %u\n", wallet_instance->GetKeyPoolSize()); tfm::format(std::cout, "Transactions: %zu\n", wallet_instance->mapWallet.size()); - tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->mapAddressBook.size()); + tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->m_address_book.size()); } bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index 0fbefb6023..cce6210129 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index ce14998fd1..ae5721bec2 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index e47e709431..75267de80b 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind aborts if can't disconnect a block. diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index ef4d9411c5..79777f5582 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for skipping signature validation on old blocks. diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index adf66243b7..75e0b93c83 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Backwards compatibility functional test diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 0c591de869..6619d83dc4 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test block processing.""" diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 073ed8d7c7..fd0330924d 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP65 (CHECKLOCKTIMEVERIFY). diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index a98480a6dd..c6852ef017 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test CSV soft fork activation. diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 38cdf0501e..05fdacd451 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP66 (DER SIG). diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 5128485ec0..3cf0fb8f7b 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index be1c044aa5..b56ffe179f 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check that it's not possible to start a second bitcoind instance using the same datadir or wallet.""" diff --git a/test/functional/feature_help.py b/test/functional/feature_help.py index e3e2456183..9f18413255 100755 --- a/test/functional/feature_help.py +++ b/test/functional/feature_help.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Verify that starting bitcoin with -h works as expected.""" diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index 7f1e8f5bae..1138b0f0ea 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test loadblock option diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py index 1bae29b302..e4bf2d849d 100755 --- a/test/functional/feature_logging.py +++ b/test/functional/feature_logging.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test debug logging.""" diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index dc00e03fe7..d4a8f8a715 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of -maxuploadtarget. diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index 9c92ee7f90..12471c5088 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test NULLDUMMY softfork. diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 21b6b299f6..c9362cf5aa 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the pruning code. diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 909a43c8d9..919d6474de 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the SegWit changeover logic.""" diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index f04a58cd19..2e80d7a248 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -1,13 +1,18 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_process_error, get_auth_cookie -class TestBitcoinCli(BitcoinTestFramework): +# The block reward of coinbaseoutput.nValue (50) BTC/block matures after +# COINBASE_MATURITY (100) blocks. Therefore, after mining 101 blocks we expect +# node 0 to have a balance of (BLOCKS - COINBASE_MATURITY) * 50 BTC/block. +BLOCKS = 101 +BALANCE = (BLOCKS - 100) * 50 +class TestBitcoinCli(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 @@ -17,15 +22,7 @@ class TestBitcoinCli(BitcoinTestFramework): def run_test(self): """Main test logic""" - - cli_response = self.nodes[0].cli("-version").send_cli() - assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response - - self.log.info("Compare responses from getwalletinfo RPC and `bitcoin-cli getwalletinfo`") - if self.is_wallet_compiled(): - cli_response = self.nodes[0].cli.getwalletinfo() - rpc_response = self.nodes[0].getwalletinfo() - assert_equal(cli_response, rpc_response) + self.nodes[0].generate(BLOCKS) self.log.info("Compare responses from getblockchaininfo RPC and `bitcoin-cli getblockchaininfo`") cli_response = self.nodes[0].cli.getblockchaininfo() @@ -35,12 +32,12 @@ class TestBitcoinCli(BitcoinTestFramework): user, password = get_auth_cookie(self.nodes[0].datadir, self.chain) self.log.info("Test -stdinrpcpass option") - assert_equal(0, self.nodes[0].cli('-rpcuser=%s' % user, '-stdinrpcpass', input=password).getblockcount()) - assert_raises_process_error(1, "Incorrect rpcuser or rpcpassword", self.nodes[0].cli('-rpcuser=%s' % user, '-stdinrpcpass', input="foo").echo) + assert_equal(BLOCKS, self.nodes[0].cli('-rpcuser={}'.format(user), '-stdinrpcpass', input=password).getblockcount()) + assert_raises_process_error(1, 'Incorrect rpcuser or rpcpassword', self.nodes[0].cli('-rpcuser={}'.format(user), '-stdinrpcpass', input='foo').echo) self.log.info("Test -stdin and -stdinrpcpass") - assert_equal(["foo", "bar"], self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input=password + "\nfoo\nbar").echo()) - assert_raises_process_error(1, "Incorrect rpcuser or rpcpassword", self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input="foo").echo) + assert_equal(['foo', 'bar'], self.nodes[0].cli('-rpcuser={}'.format(user), '-stdin', '-stdinrpcpass', input=password + '\nfoo\nbar').echo()) + assert_raises_process_error(1, 'Incorrect rpcuser or rpcpassword', self.nodes[0].cli('-rpcuser={}'.format(user), '-stdin', '-stdinrpcpass', input='foo').echo) self.log.info("Test connecting to a non-existing server") assert_raises_process_error(1, "Could not connect to the server", self.nodes[0].cli('-rpcport=1').echo) @@ -48,29 +45,47 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test connecting with non-existing RPC cookie file") assert_raises_process_error(1, "Could not locate RPC credentials", self.nodes[0].cli('-rpccookiefile=does-not-exist', '-rpcpassword=').echo) - self.log.info("Make sure that -getinfo with arguments fails") + self.log.info("Test -getinfo with arguments fails") assert_raises_process_error(1, "-getinfo takes no arguments", self.nodes[0].cli('-getinfo').help) - self.log.info("Compare responses from `bitcoin-cli -getinfo` and the RPCs data is retrieved from.") - cli_get_info = self.nodes[0].cli('-getinfo').send_cli() + self.log.info("Test -getinfo returns expected network and blockchain info") if self.is_wallet_compiled(): - wallet_info = self.nodes[0].getwalletinfo() + self.nodes[0].encryptwallet(password) + cli_get_info = self.nodes[0].cli().send_cli('-getinfo') network_info = self.nodes[0].getnetworkinfo() blockchain_info = self.nodes[0].getblockchaininfo() - assert_equal(cli_get_info['version'], network_info['version']) assert_equal(cli_get_info['blocks'], blockchain_info['blocks']) + assert_equal(cli_get_info['headers'], blockchain_info['headers']) assert_equal(cli_get_info['timeoffset'], network_info['timeoffset']) assert_equal(cli_get_info['connections'], network_info['connections']) assert_equal(cli_get_info['proxy'], network_info['networks'][0]['proxy']) assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) assert_equal(cli_get_info['chain'], blockchain_info['chain']) + if self.is_wallet_compiled(): - assert_equal(cli_get_info['balance'], wallet_info['balance']) + self.log.info("Test -getinfo and bitcoin-cli getwalletinfo return expected wallet info") + assert_equal(cli_get_info['balance'], BALANCE) + wallet_info = self.nodes[0].getwalletinfo() assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) + assert_equal(cli_get_info['unlocked_until'], wallet_info['unlocked_until']) assert_equal(cli_get_info['paytxfee'], wallet_info['paytxfee']) assert_equal(cli_get_info['relayfee'], network_info['relayfee']) - # unlocked_until is not tested because the wallet is not encrypted + assert_equal(self.nodes[0].cli.getwalletinfo(), wallet_info) + else: + self.log.info("*** Wallet not compiled; cli getwalletinfo and -getinfo wallet tests skipped") + + self.log.info("Test -version with node stopped") + self.stop_node(0) + cli_response = self.nodes[0].cli().send_cli('-version') + assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response + + self.log.info("Test -rpcwait option successfully waits for RPC connection") + self.nodes[0].start() # start node without RPC connection + self.nodes[0].wait_for_cookie_credentials() # ensure cookie file is available to avoid race condition + blocks = self.nodes[0].cli('-rpcwait').send_cli('getblockcount') + self.nodes[0].wait_for_rpc_connection() + assert_equal(blocks, BLOCKS) if __name__ == '__main__': diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index 3b3b5bb0c1..ac2e73fc5c 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests some generic aspects of the RPC interface.""" diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 9ade22a7eb..6df8f1c3ec 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool acceptance of raw transactions.""" diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index d3690b245e..e1671624a8 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool persistence. diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index d797dff134..e360b550a4 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool re-org scenarios. diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 8262e30592..1bda167c87 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mining RPCs diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py new file mode 100755 index 0000000000..6046237101 --- /dev/null +++ b/test/functional/p2p_addr_relay.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test addr relay +""" + +from test_framework.messages import ( + CAddress, + NODE_NETWORK, + NODE_WITNESS, + msg_addr, +) +from test_framework.mininode import ( + P2PInterface, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) +import time + +ADDRS = [] +for i in range(10): + addr = CAddress() + addr.time = int(time.time()) + i + addr.nServices = NODE_NETWORK | NODE_WITNESS + addr.ip = "123.123.123.{}".format(i % 256) + addr.port = 8333 + i + ADDRS.append(addr) + + +class AddrReceiver(P2PInterface): + def on_addr(self, message): + for addr in message.addrs: + assert_equal(addr.nServices, 9) + assert addr.ip.startswith('123.123.123.') + assert (8333 <= addr.port < 8343) + + +class AddrTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + + def run_test(self): + self.log.info('Create connection that sends addr messages') + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + msg = msg_addr() + + self.log.info('Send too large addr message') + msg.addrs = ADDRS * 101 + with self.nodes[0].assert_debug_log(['message addr size() = 1010']): + addr_source.send_and_ping(msg) + + self.log.info('Check that addr message content is relayed and added to addrman') + addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) + msg.addrs = ADDRS + with self.nodes[0].assert_debug_log([ + 'Added 10 addresses from 127.0.0.1: 0 tried', + 'received: addr (301 bytes) peer=0', + 'sending addr (301 bytes) peer=1', + ]): + addr_source.send_and_ping(msg) + self.nodes[0].setmocktime(int(time.time()) + 30 * 60) + addr_receiver.sync_with_ping() + + +if __name__ == '__main__': + AddrTest().main() diff --git a/test/functional/p2p_dos_header_tree.py b/test/functional/p2p_dos_header_tree.py index 28dd809ca5..f8552cf53d 100755 --- a/test/functional/p2p_dos_header_tree.py +++ b/test/functional/p2p_dos_header_tree.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that we reject low difficulty headers to prevent our block tree from filling up with useless bloat""" diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 188b130a57..3879f4ffe6 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -7,18 +7,17 @@ Test BIP 37 """ from test_framework.messages import ( + CInv, MSG_BLOCK, MSG_FILTERED_BLOCK, - msg_getdata, - msg_filterload, msg_filteradd, msg_filterclear, + msg_filterload, + msg_getdata, ) -from test_framework.mininode import ( - P2PInterface, - mininode_lock, -) +from test_framework.mininode import P2PInterface from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal class FilterNode(P2PInterface): @@ -38,8 +37,9 @@ class FilterNode(P2PInterface): for i in message.inv: # inv messages can only contain TX or BLOCK, so translate BLOCK to FILTERED_BLOCK if i.type == MSG_BLOCK: - i.type = MSG_FILTERED_BLOCK - want.inv.append(i) + want.inv.append(CInv(MSG_FILTERED_BLOCK, i.hash)) + else: + want.inv.append(i) if len(want.inv): self.send_message(want) @@ -63,24 +63,26 @@ class FilterTest(BitcoinTestFramework): self.skip_if_no_wallet() def run_test(self): - self.log.info('Add filtered P2P connection to the node') filter_node = self.nodes[0].add_p2p_connection(FilterNode()) + + self.log.info('Check that too large filter is rejected') + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=51, nTweak=0, nFlags=1)) + + self.log.info('Add filtered P2P connection to the node') filter_node.send_and_ping(filter_node.watch_filter_init) filter_address = self.nodes[0].decodescript(filter_node.watch_script_pubkey)['addresses'][0] self.log.info('Check that we receive merkleblock and tx if the filter matches a tx in a block') - filter_node.merkleblock_received = False block_hash = self.nodes[0].generatetoaddress(1, filter_address)[0] txid = self.nodes[0].getblock(block_hash)['tx'][0] + filter_node.wait_for_merkleblock(block_hash) filter_node.wait_for_tx(txid) - assert filter_node.merkleblock_received self.log.info('Check that we only receive a merkleblock if the filter does not match a tx in a block') - with mininode_lock: - filter_node.last_message.pop("merkleblock", None) filter_node.tx_received = False - self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress()) - filter_node.wait_for_merkleblock() + block_hash = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0] + filter_node.wait_for_merkleblock(block_hash) assert not filter_node.tx_received self.log.info('Check that we not receive a tx if the filter does not match a mempool tx') @@ -104,6 +106,21 @@ class FilterTest(BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 7) filter_node.wait_for_tx(txid) + self.log.info('Check that request for filtered blocks is ignored if no filter is set') + filter_node.merkleblock_received = False + filter_node.tx_received = False + with self.nodes[0].assert_debug_log(expected_msgs=['received getdata']): + block_hash = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0] + filter_node.wait_for_inv([CInv(MSG_BLOCK, int(block_hash, 16))]) + filter_node.sync_with_ping() + assert not filter_node.merkleblock_received + assert not filter_node.tx_received + + self.log.info('Check that sending "filteradd" if no filter is set is treated as misbehavior (+100)') + assert_equal(self.nodes[0].getpeerinfo()[0]['banscore'], 0) + filter_node.send_and_ping(msg_filteradd(data=b'letsmisbehave')) + assert_equal(self.nodes[0].getpeerinfo()[0]['banscore'], 100) + self.log.info("Check that division-by-zero remote crash bug [CVE-2013-5700] is fixed") filter_node.send_and_ping(msg_filterload(data=b'', nHashFuncs=1)) filter_node.send_and_ping(msg_filteradd(data=b'letstrytocrashthisnode')) diff --git a/test/functional/p2p_invalid_block.py b/test/functional/p2p_invalid_block.py index 801407757f..e280a62997 100755 --- a/test/functional/p2p_invalid_block.py +++ b/test/functional/p2p_invalid_block.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid blocks. diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index 33b7060060..0155eb21f0 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid locators. @@ -34,7 +34,7 @@ class InvalidLocatorTest(BitcoinTestFramework): msg.locator.vHave = [int(node.getblockhash(i - 1), 16) for i in range(block_count, block_count - (MAX_LOCATOR_SZ), -1)] node.p2p.send_message(msg) if type(msg) == msg_getheaders: - node.p2p.wait_for_header(int(node.getbestblockhash(), 16)) + node.p2p.wait_for_header(node.getbestblockhash()) else: node.p2p.wait_for_block(int(node.getbestblockhash(), 16)) diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 759f111e69..4bd832e8f7 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" @@ -8,14 +8,18 @@ import struct import sys from test_framework import messages -from test_framework.mininode import P2PDataStore, NetworkThread +from test_framework.mininode import ( + NetworkThread, + P2PDataStore, + P2PInterface, +) from test_framework.test_framework import BitcoinTestFramework class msg_unrecognized: """Nonsensical message. Modeled after similar types in test_framework.messages.""" - command = b'badmsg' + msgtype = b'badmsg' def __init__(self, *, str_data): self.str_data = str_data.encode() if not isinstance(str_data, bytes) else str_data @@ -24,7 +28,7 @@ class msg_unrecognized: return messages.ser_string(self.str_data) def __repr__(self): - return "{}(data={})".format(self.command, self.str_data) + return "{}(data={})".format(self.msgtype, self.str_data) class InvalidMessagesTest(BitcoinTestFramework): @@ -46,7 +50,8 @@ class InvalidMessagesTest(BitcoinTestFramework): self.test_magic_bytes() self.test_checksum() self.test_size() - self.test_command() + self.test_msgtype() + self.test_large_inv() node = self.nodes[0] self.node = node @@ -163,7 +168,7 @@ class InvalidMessagesTest(BitcoinTestFramework): msg = conn.build_message(msg_unrecognized(str_data="d")) cut_len = ( 4 + # magic - 12 + # command + 12 + # msgtype 4 #len ) # modify checksum @@ -178,7 +183,7 @@ class InvalidMessagesTest(BitcoinTestFramework): msg = conn.build_message(msg_unrecognized(str_data="d")) cut_len = ( 4 + # magic - 12 # command + 12 # msgtype ) # modify len to MAX_SIZE + 1 msg = msg[:cut_len] + struct.pack("<I", 0x02000000 + 1) + msg[cut_len + 4:] @@ -186,18 +191,31 @@ class InvalidMessagesTest(BitcoinTestFramework): conn.wait_for_disconnect(timeout=1) self.nodes[0].disconnect_p2ps() - def test_command(self): + def test_msgtype(self): conn = self.nodes[0].add_p2p_connection(P2PDataStore()) with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: ERRORS IN HEADER']): msg = msg_unrecognized(str_data="d") - msg.command = b'\xff' * 12 + msg.msgtype = b'\xff' * 12 msg = conn.build_message(msg) - # Modify command + # Modify msgtype msg = msg[:7] + b'\x00' + msg[7 + 1:] self.nodes[0].p2p.send_raw_message(msg) conn.sync_with_ping(timeout=1) self.nodes[0].disconnect_p2ps() + def test_large_inv(self): + conn = self.nodes[0].add_p2p_connection(P2PInterface()) + with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (0 -> 20): message inv size() = 50001']): + msg = messages.msg_inv([messages.CInv(1, 1)] * 50001) + conn.send_and_ping(msg) + with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (20 -> 40): message getdata size() = 50001']): + msg = messages.msg_getdata([messages.CInv(1, 1)] * 50001) + conn.send_and_ping(msg) + with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (40 -> 60): headers message size = 2001']): + msg = messages.msg_headers([messages.CBlockHeader()] * 2001) + conn.send_and_ping(msg) + self.nodes[0].disconnect_p2ps() + def _tweak_msg_data_size(self, message, wrong_size): """ Return a raw message based on another message but with an incorrect data size in diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 5975a52b2a..c70a892463 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid transactions. @@ -118,6 +118,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): tx_orphan_2_invalid = CTransaction() tx_orphan_2_invalid.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_1.sha256, 2))) tx_orphan_2_invalid.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_2_invalid.calc_sha256() self.log.info('Send the orphans ... ') # Send valid orphan txs from p2ps[0] @@ -148,6 +149,22 @@ class InvalidTxRequestTest(BitcoinTestFramework): wait_until(lambda: 1 == len(node.getpeerinfo()), timeout=12) # p2ps[1] is no longer connected assert_equal(expected_mempool, set(node.getrawmempool())) + self.log.info('Test orphan pool overflow') + orphan_tx_pool = [CTransaction() for _ in range(101)] + for i in range(len(orphan_tx_pool)): + orphan_tx_pool[i].vin.append(CTxIn(outpoint=COutPoint(i, 333))) + orphan_tx_pool[i].vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + + with node.assert_debug_log(['mapOrphan overflow, removed 1 tx']): + node.p2p.send_txs_and_test(orphan_tx_pool, node, success=False) + + rejected_parent = CTransaction() + rejected_parent.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_2_invalid.sha256, 0))) + rejected_parent.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + rejected_parent.rehash() + with node.assert_debug_log(['not keeping orphan with rejected parents {}'.format(rejected_parent.hash)]): + node.p2p.send_txs_and_test([rejected_parent], node, success=False) + if __name__ == '__main__': InvalidTxRequestTest().main() diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 2751ae6a5b..7f7430d04e 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test message sending before handshake completion. -A node should never send anything other than VERSION/VERACK/REJECT until it's +A node should never send anything other than VERSION/VERACK until it's received a VERACK. This test connects to a node and sends it a few messages, trying to entice it @@ -12,10 +12,19 @@ into sending us something it shouldn't.""" import time -from test_framework.messages import msg_getaddr, msg_ping, msg_verack +from test_framework.messages import ( + msg_getaddr, + msg_ping, + msg_verack, + msg_version, +) from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import wait_until +from test_framework.util import ( + assert_equal, + assert_greater_than_or_equal, + wait_until, +) banscore = 10 @@ -28,14 +37,13 @@ class CLazyNode(P2PInterface): def bad_message(self, message): self.unexpected_msg = True - self.log.info("should not have received message: %s" % message.command) + self.log.info("should not have received message: %s" % message.msgtype) def on_open(self): self.ever_connected = True def on_version(self, message): self.bad_message(message) def on_verack(self, message): self.bad_message(message) - def on_reject(self, message): self.bad_message(message) def on_inv(self, message): self.bad_message(message) def on_addr(self, message): self.bad_message(message) def on_getdata(self, message): self.bad_message(message) @@ -65,8 +73,6 @@ class CNodeNoVersionBan(CLazyNode): for i in range(banscore): self.send_message(msg_verack()) - def on_reject(self, message): pass - # Node that never sends a version. This one just sits idle and hopes to receive # any message (it shouldn't!) class CNodeNoVersionIdle(CLazyNode): @@ -79,7 +85,6 @@ class CNodeNoVerackIdle(CLazyNode): self.version_received = False super().__init__() - def on_reject(self, message): pass def on_verack(self, message): pass # When version is received, don't reply with a verack. Instead, see if the # node will give us a message that it shouldn't. This is not an exhaustive @@ -90,6 +95,14 @@ class CNodeNoVerackIdle(CLazyNode): self.send_message(msg_getaddr()) +class P2PVersionStore(P2PInterface): + version_received = None + + def on_version(self, msg): + super().on_version(msg) + self.version_received = msg + + class P2PLeakTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -127,6 +140,27 @@ class P2PLeakTest(BitcoinTestFramework): assert no_version_idlenode.unexpected_msg == False assert no_verack_idlenode.unexpected_msg == False + self.log.info('Check that the version message does not leak the local address of the node') + time_begin = int(time.time()) + p2p_version_store = self.nodes[0].add_p2p_connection(P2PVersionStore()) + time_end = time.time() + ver = p2p_version_store.version_received + assert_greater_than_or_equal(ver.nTime, time_begin) + assert_greater_than_or_equal(time_end, ver.nTime) + assert_equal(ver.addrFrom.port, 0) + assert_equal(ver.addrFrom.ip, '0.0.0.0') + assert_equal(ver.nStartingHeight, 201) + assert_equal(ver.nRelay, 1) + + self.log.info('Check that old nodes are disconnected') + p2p_old_node = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) + old_version_msg = msg_version() + old_version_msg.nVersion = 31799 + wait_until(lambda: p2p_old_node.is_connected) + with self.nodes[0].assert_debug_log(['peer=4 using obsolete version 31799; disconnecting']): + p2p_old_node.send_message(old_version_msg) + p2p_old_node.wait_for_disconnect() + if __name__ == '__main__': P2PLeakTest().main() diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index c6fcc5e200..6b3436fa5f 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that we don't leak txs to inbound peers that we haven't yet announced to""" diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 3a7bf4bfc3..2c200fccad 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test p2p permission message. diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index d8dce7fe56..09089371d3 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test segwit transactions and blocks on P2P network.""" diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 84d818400a..74d5536f5f 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of headers messages to announce blocks. diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index c1235f8a6b..5a4fa42988 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various net timeouts. diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index b56dc994e7..a999fba818 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ @@ -152,6 +152,10 @@ class TxDownloadTest(BitcoinTestFramework): wait_until(lambda: p.tx_getdata_count == MAX_GETDATA_IN_FLIGHT + 2) self.nodes[0].setmocktime(0) + def test_spurious_notfound(self): + self.log.info('Check that spurious notfound is ignored') + self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(1, 1)])) + def run_test(self): # Setup the p2p connections self.peers = [] @@ -161,6 +165,8 @@ class TxDownloadTest(BitcoinTestFramework): self.log.info("Nodes are setup with {} incoming connections each".format(NUM_INBOUND)) + self.test_spurious_notfound() + # Test the in-flight max first, because we want no transactions in # flight ahead of this test. self.test_in_flight_max() diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index d18d4b069a..3aaf4b9977 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index a6eaaa4539..6273c229ae 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPCs related to blockchainstate. diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 438e7f68e0..e65787ce08 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the generation of UTXO snapshots using `dumptxoutset`. diff --git a/test/functional/rpc_estimatefee.py b/test/functional/rpc_estimatefee.py index 8bdecfc8cd..1fff9e1512 100755 --- a/test/functional/rpc_estimatefee.py +++ b/test/functional/rpc_estimatefee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the estimatefee RPCs. diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index c435ef24ce..4bc4913bda 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the fundrawtransaction RPC.""" @@ -500,11 +500,16 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[1].getnewaddress() self.nodes[1].getrawchangeaddress() inputs = [] - outputs = {self.nodes[0].getnewaddress():1.1} + outputs = {self.nodes[0].getnewaddress():1.09999500} rawtx = self.nodes[1].createrawtransaction(inputs, outputs) + # fund a transaction that does not require a new key for the change output + self.nodes[1].fundrawtransaction(rawtx) + # fund a transaction that requires a new key for the change output # creating the key must be impossible because the wallet is locked - assert_raises_rpc_error(-4, "Keypool ran out, please call keypoolrefill first", self.nodes[1].fundrawtransaction, rawtx) + outputs = {self.nodes[0].getnewaddress():1.1} + rawtx = self.nodes[1].createrawtransaction(inputs, outputs) + assert_raises_rpc_error(-4, "Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.", self.nodes[1].fundrawtransaction, rawtx) # Refill the keypool. self.nodes[1].walletpassphrase("test", 100) diff --git a/test/functional/rpc_generateblock.py b/test/functional/rpc_generateblock.py new file mode 100755 index 0000000000..aa58c0af9d --- /dev/null +++ b/test/functional/rpc_generateblock.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +'''Test generateblock rpc. +''' + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + + +class GenerateBlockTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + node = self.nodes[0] + + self.log.info('Generate an empty block to address') + address = node.getnewaddress() + hash = node.generateblock(output=address, transactions=[])['hash'] + block = node.getblock(blockhash=hash, verbose=2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], address) + + self.log.info('Generate an empty block to a descriptor') + hash = node.generateblock('addr(' + address + ')', [])['hash'] + block = node.getblock(blockhash=hash, verbosity=2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], address) + + self.log.info('Generate an empty block to a combo descriptor with compressed pubkey') + combo_key = '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' + combo_address = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' + hash = node.generateblock('combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], combo_address) + + self.log.info('Generate an empty block to a combo descriptor with uncompressed pubkey') + combo_key = '0408ef68c46d20596cc3f6ddf7c8794f71913add807f1dc55949fa805d764d191c0b7ce6894c126fce0babc6663042f3dde9b0cf76467ea315514e5a6731149c67' + combo_address = 'mkc9STceoCcjoXEXe6cm66iJbmjM6zR9B2' + hash = node.generateblock('combo(' + combo_key + ')', [])['hash'] + block = node.getblock(hash, 2) + assert_equal(len(block['tx']), 1) + assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['addresses'][0], combo_address) + + # Generate 110 blocks to spend + node.generatetoaddress(110, address) + + # Generate some extra mempool transactions to verify they don't get mined + for i in range(10): + node.sendtoaddress(address, 0.001) + + self.log.info('Generate block with txid') + txid = node.sendtoaddress(address, 1) + hash = node.generateblock(address, [txid])['hash'] + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + assert_equal(block['tx'][1], txid) + + self.log.info('Generate block with raw tx') + utxos = node.listunspent(addresses=[address]) + raw = node.createrawtransaction([{'txid':utxos[0]['txid'], 'vout':utxos[0]['vout']}],[{address:1}]) + signed_raw = node.signrawtransactionwithwallet(raw)['hex'] + hash = node.generateblock(address, [signed_raw])['hash'] + block = node.getblock(hash, 1) + assert_equal(len(block['tx']), 2) + txid = block['tx'][1] + assert_equal(node.gettransaction(txid)['hex'], signed_raw) + + self.log.info('Fail to generate block with out of order txs') + raw1 = node.createrawtransaction([{'txid':txid, 'vout':0}],[{address:0.9999}]) + signed_raw1 = node.signrawtransactionwithwallet(raw1)['hex'] + txid1 = node.sendrawtransaction(signed_raw1) + raw2 = node.createrawtransaction([{'txid':txid1, 'vout':0}],[{address:0.999}]) + signed_raw2 = node.signrawtransactionwithwallet(raw2)['hex'] + assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', node.generateblock, address, [signed_raw2, txid1]) + + self.log.info('Fail to generate block with txid not in mempool') + missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' + assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', node.generateblock, address, [missing_txid]) + + self.log.info('Fail to generate block with invalid raw tx') + invalid_raw_tx = '0000' + assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, node.generateblock, address, [invalid_raw_tx]) + + self.log.info('Fail to generate block with invalid address/descriptor') + assert_raises_rpc_error(-5, 'Invalid address or descriptor', node.generateblock, '1234', []) + + self.log.info('Fail to generate block with a ranged descriptor') + ranged_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0/*)' + assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', node.generateblock, ranged_descriptor, []) + + self.log.info('Fail to generate block with a descriptor missing a private key') + child_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0\'/0)' + assert_raises_rpc_error(-5, 'Cannot derive script without private keys', node.generateblock, child_descriptor, []) + +if __name__ == '__main__': + GenerateBlockTest().main() diff --git a/test/functional/rpc_getaddressinfo_label_deprecation.py b/test/functional/rpc_getaddressinfo_label_deprecation.py index 5e739ebede..4c6b2fe5cf 100755 --- a/test/functional/rpc_getaddressinfo_label_deprecation.py +++ b/test/functional/rpc_getaddressinfo_label_deprecation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py b/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py index 3f2e8dee18..903f5536b9 100755 --- a/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py +++ b/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020-2019 The Bitcoin Core developers +# Copyright (c) 2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 3a63377545..ea8510f92b 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the Partially Signed Transaction RPCs. diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index c3d34be0dd..861b394e70 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the scantxoutset rpc call.""" diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index 780758e219..3d08202724 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction signing using the signrawtransaction* RPCs.""" +from test_framework.address import check_script, script_to_p2sh from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, hex_str_to_bytes +from test_framework.util import assert_equal, assert_raises_rpc_error, find_vout_for_address, hex_str_to_bytes from test_framework.messages import sha256 -from test_framework.script import CScript, OP_0 +from test_framework.script import CScript, OP_0, OP_CHECKSIG from decimal import Decimal @@ -26,6 +27,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): 1) The transaction has a complete set of signatures 2) No script verification error occurred""" + self.log.info("Test valid raw transaction with one input") privKeys = ['cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N', 'cVKpPfVKSJxKqVpE9awvXNWuLHCa5j5tiE7K6zbUSptFpTEtiFrA'] inputs = [ @@ -48,7 +50,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert 'errors' not in rawTxSigned def test_with_lock_outputs(self): - """Test correct error reporting when trying to sign a locked output""" + self.log.info("Test correct error reporting when trying to sign a locked output") self.nodes[0].encryptwallet("password") rawTx = '020000000156b958f78e3f24e0b2f4e4db1255426b0902027cb37e3ddadb52e37c3557dddb0000000000ffffffff01c0a6b929010000001600149a2ee8c77140a053f36018ac8124a6ececc1668a00000000' @@ -64,6 +66,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): 4) Two script verification errors occurred 5) Script verification errors have certain properties ("txid", "vout", "scriptSig", "sequence", "error") 6) The verification errors refer to the invalid (vin 1) and missing input (vin 2)""" + self.log.info("Test script verification errors") privKeys = ['cUeKHd5orzT3mz8P9pxyREHfsWtVfgsfDjiZZBcjUBAaGk1BTj7N'] inputs = [ @@ -146,7 +149,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert not rawTxSigned['errors'][0]['witness'] def witness_script_test(self): - # Now test signing transaction to P2SH-P2WSH addresses without wallet + self.log.info("Test signing transaction to P2SH-P2WSH addresses without wallet") # Create a new P2SH-P2WSH 1-of-1 multisig address: embedded_address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress()) embedded_privkey = self.nodes[1].dumpprivkey(embedded_address["address"]) @@ -168,6 +171,33 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert 'complete' in spending_tx_signed assert_equal(spending_tx_signed['complete'], True) + # Now test with P2PKH and P2PK scripts as the witnessScript + for tx_type in ['P2PKH', 'P2PK']: # these tests are order-independent + self.verify_txn_with_witness_script(tx_type) + + def verify_txn_with_witness_script(self, tx_type): + self.log.info("Test with a {} script as the witnessScript".format(tx_type)) + embedded_addr_info = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress('', 'legacy')) + embedded_privkey = self.nodes[1].dumpprivkey(embedded_addr_info['address']) + witness_script = { + 'P2PKH': embedded_addr_info['scriptPubKey'], + 'P2PK': CScript([hex_str_to_bytes(embedded_addr_info['pubkey']), OP_CHECKSIG]).hex() + }.get(tx_type, "Invalid tx_type") + redeem_script = CScript([OP_0, sha256(check_script(witness_script))]).hex() + addr = script_to_p2sh(redeem_script) + script_pub_key = self.nodes[1].validateaddress(addr)['scriptPubKey'] + # Fund that address + txid = self.nodes[0].sendtoaddress(addr, 10) + vout = find_vout_for_address(self.nodes[0], txid, addr) + self.nodes[0].generate(1) + # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys + spending_tx = self.nodes[0].createrawtransaction([{'txid': txid, 'vout': vout}], {self.nodes[1].getnewaddress(): Decimal("9.999")}) + spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [{'txid': txid, 'vout': vout, 'scriptPubKey': script_pub_key, 'redeemScript': redeem_script, 'witnessScript': witness_script, 'amount': 10}]) + # Check the signing completed successfully + assert 'complete' in spending_tx_signed + assert_equal(spending_tx_signed['complete'], True) + self.nodes[0].sendrawtransaction(spending_tx_signed['hex']) + def run_test(self): self.successful_signing_test() self.script_verification_error_test() diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 6a7e91216a..8f410f233e 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Encode and decode BASE58, P2PKH and P2SH addresses.""" diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 4ba6ac1db2..05308931e3 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -101,23 +101,26 @@ class AuthServiceProxy(): if os.name == 'nt': # Windows somehow does not like to re-use connections # TODO: Find out why the connection would disconnect occasionally and make it reusable on Windows + # Avoid "ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine" self._set_conn() try: self.__conn.request(method, path, postdata, headers) return self._get_response() - except http.client.BadStatusLine as e: - if e.line == "''": # if connection was closed, try again + except (BrokenPipeError, ConnectionResetError): + # Python 3.5+ raises BrokenPipeError when the connection was reset + # ConnectionResetError happens on FreeBSD + self.__conn.close() + self.__conn.request(method, path, postdata, headers) + return self._get_response() + except OSError as e: + retry = ( + '[WinError 10053] An established connection was aborted by the software in your host machine' in str(e)) + if retry: self.__conn.close() self.__conn.request(method, path, postdata, headers) return self._get_response() else: raise - except (BrokenPipeError, ConnectionResetError): - # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset - # ConnectionResetError happens on FreeBSD with Python 3.4 - self.__conn.close() - self.__conn.request(method, path, postdata, headers) - return self._get_response() def get_request(self, *args, **argsn): AuthServiceProxy.__id_count += 1 diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 5f8fcc6fd8..cf86e1bdf7 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2019 The Bitcoin Core developers +# Copyright (c) 2010-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin test framework primitive and message structures @@ -946,7 +946,7 @@ class CMerkleBlock: class msg_version: __slots__ = ("addrFrom", "addrTo", "nNonce", "nRelay", "nServices", "nStartingHeight", "nTime", "nVersion", "strSubVer") - command = b"version" + msgtype = b"version" def __init__(self): self.nVersion = MY_VERSION @@ -1004,7 +1004,7 @@ class msg_version: class msg_verack: __slots__ = () - command = b"verack" + msgtype = b"verack" def __init__(self): pass @@ -1021,7 +1021,7 @@ class msg_verack: class msg_addr: __slots__ = ("addrs",) - command = b"addr" + msgtype = b"addr" def __init__(self): self.addrs = [] @@ -1038,7 +1038,7 @@ class msg_addr: class msg_inv: __slots__ = ("inv",) - command = b"inv" + msgtype = b"inv" def __init__(self, inv=None): if inv is None: @@ -1058,7 +1058,7 @@ class msg_inv: class msg_getdata: __slots__ = ("inv",) - command = b"getdata" + msgtype = b"getdata" def __init__(self, inv=None): self.inv = inv if inv is not None else [] @@ -1075,7 +1075,7 @@ class msg_getdata: class msg_getblocks: __slots__ = ("locator", "hashstop") - command = b"getblocks" + msgtype = b"getblocks" def __init__(self): self.locator = CBlockLocator() @@ -1099,7 +1099,7 @@ class msg_getblocks: class msg_tx: __slots__ = ("tx",) - command = b"tx" + msgtype = b"tx" def __init__(self, tx=CTransaction()): self.tx = tx @@ -1123,7 +1123,7 @@ class msg_no_witness_tx(msg_tx): class msg_block: __slots__ = ("block",) - command = b"block" + msgtype = b"block" def __init__(self, block=None): if block is None: @@ -1142,12 +1142,12 @@ class msg_block: # for cases where a user needs tighter control over what is sent over the wire -# note that the user must supply the name of the command, and the data +# note that the user must supply the name of the msgtype, and the data class msg_generic: - __slots__ = ("command", "data") + __slots__ = ("msgtype", "data") - def __init__(self, command, data=None): - self.command = command + def __init__(self, msgtype, data=None): + self.msgtype = msgtype self.data = data def serialize(self): @@ -1165,7 +1165,7 @@ class msg_no_witness_block(msg_block): class msg_getaddr: __slots__ = () - command = b"getaddr" + msgtype = b"getaddr" def __init__(self): pass @@ -1182,7 +1182,7 @@ class msg_getaddr: class msg_ping: __slots__ = ("nonce",) - command = b"ping" + msgtype = b"ping" def __init__(self, nonce=0): self.nonce = nonce @@ -1201,7 +1201,7 @@ class msg_ping: class msg_pong: __slots__ = ("nonce",) - command = b"pong" + msgtype = b"pong" def __init__(self, nonce=0): self.nonce = nonce @@ -1220,7 +1220,7 @@ class msg_pong: class msg_mempool: __slots__ = () - command = b"mempool" + msgtype = b"mempool" def __init__(self): pass @@ -1237,7 +1237,7 @@ class msg_mempool: class msg_notfound: __slots__ = ("vec", ) - command = b"notfound" + msgtype = b"notfound" def __init__(self, vec=None): self.vec = vec or [] @@ -1254,7 +1254,7 @@ class msg_notfound: class msg_sendheaders: __slots__ = () - command = b"sendheaders" + msgtype = b"sendheaders" def __init__(self): pass @@ -1275,7 +1275,7 @@ class msg_sendheaders: # hash_stop (hash of last desired block header, 0 to get as many as possible) class msg_getheaders: __slots__ = ("hashstop", "locator",) - command = b"getheaders" + msgtype = b"getheaders" def __init__(self): self.locator = CBlockLocator() @@ -1301,7 +1301,7 @@ class msg_getheaders: # <count> <vector of block headers> class msg_headers: __slots__ = ("headers",) - command = b"headers" + msgtype = b"headers" def __init__(self, headers=None): self.headers = headers if headers is not None else [] @@ -1321,15 +1321,28 @@ class msg_headers: class msg_merkleblock: - command = b"merkleblock" + __slots__ = ("merkleblock",) + msgtype = b"merkleblock" + + def __init__(self, merkleblock=None): + if merkleblock is None: + self.merkleblock = CMerkleBlock() + else: + self.merkleblock = merkleblock def deserialize(self, f): - pass # Placeholder for now + self.merkleblock.deserialize(f) + + def serialize(self): + return self.merkleblock.serialize() + + def __repr__(self): + return "msg_merkleblock(merkleblock=%s)" % (repr(self.merkleblock)) class msg_filterload: __slots__ = ("data", "nHashFuncs", "nTweak", "nFlags") - command = b"filterload" + msgtype = b"filterload" def __init__(self, data=b'00', nHashFuncs=0, nTweak=0, nFlags=0): self.data = data @@ -1358,7 +1371,7 @@ class msg_filterload: class msg_filteradd: __slots__ = ("data") - command = b"filteradd" + msgtype = b"filteradd" def __init__(self, data): self.data = data @@ -1377,7 +1390,7 @@ class msg_filteradd: class msg_filterclear: __slots__ = () - command = b"filterclear" + msgtype = b"filterclear" def __init__(self): pass @@ -1394,7 +1407,7 @@ class msg_filterclear: class msg_feefilter: __slots__ = ("feerate",) - command = b"feefilter" + msgtype = b"feefilter" def __init__(self, feerate=0): self.feerate = feerate @@ -1413,7 +1426,7 @@ class msg_feefilter: class msg_sendcmpct: __slots__ = ("announce", "version") - command = b"sendcmpct" + msgtype = b"sendcmpct" def __init__(self): self.announce = False @@ -1435,7 +1448,7 @@ class msg_sendcmpct: class msg_cmpctblock: __slots__ = ("header_and_shortids",) - command = b"cmpctblock" + msgtype = b"cmpctblock" def __init__(self, header_and_shortids = None): self.header_and_shortids = header_and_shortids @@ -1455,7 +1468,7 @@ class msg_cmpctblock: class msg_getblocktxn: __slots__ = ("block_txn_request",) - command = b"getblocktxn" + msgtype = b"getblocktxn" def __init__(self): self.block_txn_request = None @@ -1475,7 +1488,7 @@ class msg_getblocktxn: class msg_blocktxn: __slots__ = ("block_transactions",) - command = b"blocktxn" + msgtype = b"blocktxn" def __init__(self): self.block_transactions = BlockTransactions() diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index ad330f2a93..ea078fd81c 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2019 The Bitcoin Core developers +# Copyright (c) 2010-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin P2P network half-a-node. @@ -180,7 +180,7 @@ class P2PConnection(asyncio.Protocol): raise ValueError("magic bytes mismatch: {} != {}".format(repr(self.magic_bytes), repr(self.recvbuf))) if len(self.recvbuf) < 4 + 12 + 4 + 4: return - command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] + msgtype = self.recvbuf[4:4+12].split(b"\x00", 1)[0] msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] checksum = self.recvbuf[4+12+4:4+12+4+4] if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen: @@ -191,10 +191,10 @@ class P2PConnection(asyncio.Protocol): if checksum != h[:4]: raise ValueError("got bad checksum " + repr(self.recvbuf)) self.recvbuf = self.recvbuf[4+12+4+4+msglen:] - if command not in MESSAGEMAP: - raise ValueError("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) + if msgtype not in MESSAGEMAP: + raise ValueError("Received unknown msgtype from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, msgtype, repr(msg))) f = BytesIO(msg) - t = MESSAGEMAP[command]() + t = MESSAGEMAP[msgtype]() t.deserialize(f) self._log_message("receive", t) self.on_message(t) @@ -233,11 +233,11 @@ class P2PConnection(asyncio.Protocol): def build_message(self, message): """Build a serialized P2P message""" - command = message.command + msgtype = message.msgtype data = message.serialize() tmsg = self.magic_bytes - tmsg += command - tmsg += b"\x00" * (12 - len(command)) + tmsg += msgtype + tmsg += b"\x00" * (12 - len(msgtype)) tmsg += struct.pack("<I", len(data)) th = sha256(data) h = sha256(th) @@ -304,10 +304,10 @@ class P2PInterface(P2PConnection): and the most recent message of each type.""" with mininode_lock: try: - command = message.command.decode('ascii') - self.message_count[command] += 1 - self.last_message[command] = message - getattr(self, 'on_' + command)(message) + msgtype = message.msgtype.decode('ascii') + self.message_count[msgtype] += 1 + self.last_message[msgtype] = message + getattr(self, 'on_' + msgtype)(message) except: print("ERROR delivering %s (%s)" % (repr(message), sys.exc_info()[0])) raise @@ -339,7 +339,6 @@ class P2PInterface(P2PConnection): def on_merkleblock(self, message): pass def on_notfound(self, message): pass def on_pong(self, message): pass - def on_reject(self, message): pass def on_sendcmpct(self, message): pass def on_sendheaders(self, message): pass def on_tx(self, message): pass @@ -393,18 +392,17 @@ class P2PInterface(P2PConnection): last_headers = self.last_message.get('headers') if not last_headers: return False - return last_headers.headers[0].rehash() == blockhash + return last_headers.headers[0].rehash() == int(blockhash, 16) wait_until(test_function, timeout=timeout, lock=mininode_lock) - def wait_for_merkleblock(self, timeout=60): + def wait_for_merkleblock(self, blockhash, timeout=60): def test_function(): assert self.is_connected last_filtered_block = self.last_message.get('merkleblock') if not last_filtered_block: return False - # TODO change this method to take a hash value and only return true if the correct block has been received - return True + return last_filtered_block.merkleblock.header.rehash() == int(blockhash, 16) wait_until(test_function, timeout=timeout, lock=mininode_lock) diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 92725dfcf4..e587a77f64 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Functionality to build scripts, as well as signature hash functions. diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index e36fb350c6..d4cf5f8896 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -549,9 +549,13 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): cwd=self.options.tmpdir, )) self.start_node(CACHE_NODE_ID) + cache_node = self.nodes[CACHE_NODE_ID] # Wait for RPC connections to be ready - self.nodes[CACHE_NODE_ID].wait_for_rpc_connection() + cache_node.wait_for_rpc_connection() + + # Set a time in the past, so that blocks don't end up in the future + cache_node.setmocktime(cache_node.getblockheader(cache_node.getbestblockhash())['time']) # Create a 199-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. @@ -560,12 +564,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # This is needed so that we are out of IBD when the test starts, # see the tip age check in IsInitialBlockDownload(). for i in range(8): - self.nodes[CACHE_NODE_ID].generatetoaddress( + cache_node.generatetoaddress( nblocks=25 if i != 7 else 24, address=TestNode.PRIV_KEYS[i % 4].address, ) - assert_equal(self.nodes[CACHE_NODE_ID].getblockchaininfo()["blocks"], 199) + assert_equal(cache_node.getblockchaininfo()["blocks"], 199) # Shut it down, and clean up cache directories: self.stop_nodes() diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 8260c917fe..64f39b8cfe 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Class for bitcoind node under test""" @@ -26,6 +26,7 @@ from .util import ( MAX_NODES, append_config, delete_cookie_file, + get_auth_cookie, get_rpc_proxy, rpc_url, wait_until, @@ -225,19 +226,38 @@ class TestNode(): self.rpc_connected = True self.url = self.rpc.url return - except IOError as e: - if e.errno != errno.ECONNREFUSED: # Port not yet open? - raise # unknown IO error except JSONRPCException as e: # Initialization phase # -28 RPC in warmup # -342 Service unavailable, RPC server started but is shutting down due to error if e.error['code'] != -28 and e.error['code'] != -342: raise # unknown JSON RPC exception - except ValueError as e: # cookie file not found and no rpcuser or rpcassword. bitcoind still starting + except ConnectionResetError: + # This might happen when the RPC server is in warmup, but shut down before the call to getblockcount + # succeeds. Try again to properly raise the FailedToStartError + pass + except OSError as e: + if e.errno != errno.ECONNREFUSED: # Port not yet open? + raise # unknown OS error + except ValueError as e: # cookie file not found and no rpcuser or rpcpassword; bitcoind is still starting if "No RPC credentials" not in str(e): raise time.sleep(1.0 / poll_per_s) - self._raise_assertion_error("Unable to connect to bitcoind") + self._raise_assertion_error("Unable to connect to bitcoind after {}s".format(self.rpc_timeout)) + + def wait_for_cookie_credentials(self): + """Ensures auth cookie credentials can be read, e.g. for testing CLI with -rpcwait before RPC connection is up.""" + self.log.debug("Waiting for cookie credentials") + # Poll at a rate of four times per second. + poll_per_s = 4 + for _ in range(poll_per_s * self.rpc_timeout): + try: + get_auth_cookie(self.datadir, self.chain) + self.log.debug("Cookie credentials successfully retrieved") + return + except ValueError: # cookie file not found and no rpcuser or rpcpassword; bitcoind is still starting + pass # so we continue polling until RPC credentials are retrieved + time.sleep(1.0 / poll_per_s) + self._raise_assertion_error("Unable to retrieve cookie credentials after {}s".format(self.rpc_timeout)) def generate(self, nblocks, maxtries=1000000): self.log.debug("TestNode.generate() dispatches `generate` call to `generatetoaddress`") @@ -523,7 +543,6 @@ def arg_to_cli(arg): class TestNodeCLI(): """Interface to bitcoin-cli for an individual node""" - def __init__(self, binary, datadir): self.options = [] self.binary = binary diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index e89b4e9879..9d5ff63f34 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helpful routines for regression testing.""" diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 2f307750a9..4f902717c3 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run regression test suite. @@ -144,6 +144,7 @@ BASE_SCRIPTS = [ 'rpc_blockchain.py', 'rpc_deprecated.py', 'wallet_disable.py', + 'p2p_addr_relay.py', 'rpc_net.py', 'wallet_keypool.py', 'p2p_mempool.py', @@ -172,6 +173,7 @@ BASE_SCRIPTS = [ 'wallet_importprunedfunds.py', 'p2p_leak_tx.py', 'rpc_signmessage.py', + 'rpc_generateblock.py', 'wallet_balance.py', 'feature_nulldummy.py', 'mempool_accept.py', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index d2629ff1ed..b3d496dd51 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-wallet.""" diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 1122daaf83..c7b19081c8 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the abandontransaction RPC. @@ -110,8 +110,8 @@ class AbandonConflictTest(BitcoinTestFramework): assert_equal(newbalance, balance - signed3_change) # Unconfirmed received funds that are not in mempool, also shouldn't show # up in unconfirmed balance - unconfbalance = self.nodes[0].getunconfirmedbalance() + self.nodes[0].getbalance() - assert_equal(unconfbalance, newbalance) + balances = self.nodes[0].getbalances()['mine'] + assert_equal(balances['untrusted_pending'] + balances['trusted'], newbalance) # Also shouldn't show up in listunspent assert not txABC2 in [utxo["txid"] for utxo in self.nodes[0].listunspent(0)] balance = newbalance diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index 79b6db986b..68e22b7e86 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet can send and receive using all combinations of address types. @@ -97,12 +97,9 @@ class AddressTypeTest(BitcoinTestFramework): connect_nodes(self.nodes[i], j) self.sync_all() - def get_balances(self, confirmed=True): - """Return a list of confirmed or unconfirmed balances.""" - if confirmed: - return [self.nodes[i].getbalance() for i in range(4)] - else: - return [self.nodes[i].getunconfirmedbalance() for i in range(4)] + def get_balances(self, key='trusted'): + """Return a list of balances.""" + return [self.nodes[i].getbalances()['mine'][key] for i in range(4)] # Quick test of python bech32 implementation def test_python_bech32(self, addr): @@ -307,7 +304,7 @@ class AddressTypeTest(BitcoinTestFramework): self.nodes[from_node].sendmany("", sends) self.sync_mempools() - unconf_balances = self.get_balances(False) + unconf_balances = self.get_balances('untrusted_pending') self.log.debug("Check unconfirmed balances: {}".format(unconf_balances)) assert_equal(unconf_balances[from_node], 0) for n, to_node in enumerate(range(from_node + 1, from_node + 4)): diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 8e2dc03ac2..78a51a1d5f 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -83,16 +83,21 @@ class AvoidReuseTest(BitcoinTestFramework): self.nodes[0].generate(110) self.sync_all() + self.test_change_remains_change(self.nodes[1]) reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) - self.test_fund_send_fund_senddirty() + self.test_sending_from_reused_address_without_avoid_reuse() reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) - self.test_fund_send_fund_send("legacy") + self.test_sending_from_reused_address_fails("legacy") reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) - self.test_fund_send_fund_send("p2sh-segwit") + self.test_sending_from_reused_address_fails("p2sh-segwit") reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) - self.test_fund_send_fund_send("bech32") + self.test_sending_from_reused_address_fails("bech32") reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) self.test_getbalances_used() + reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) + self.test_full_destination_group_is_preferred() + reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) + self.test_all_destination_groups_are_used() def test_persistence(self): '''Test that wallet files persist the avoid_reuse flag.''' @@ -137,13 +142,37 @@ class AvoidReuseTest(BitcoinTestFramework): # Unload temp wallet self.nodes[1].unloadwallet(tempwallet) - def test_fund_send_fund_senddirty(self): + def test_change_remains_change(self, node): + self.log.info("Test that change doesn't turn into non-change when spent") + + reset_balance(node, node.getnewaddress()) + addr = node.getnewaddress() + txid = node.sendtoaddress(addr, 1) + out = node.listunspent(minconf=0, query_options={'minimumAmount': 2}) + assert_equal(len(out), 1) + assert_equal(out[0]['txid'], txid) + changeaddr = out[0]['address'] + + # Make sure it's starting out as change as expected + assert node.getaddressinfo(changeaddr)['ischange'] + for logical_tx in node.listtransactions(): + assert logical_tx.get('address') != changeaddr + + # Spend it + reset_balance(node, node.getnewaddress()) + + # It should still be change + assert node.getaddressinfo(changeaddr)['ischange'] + for logical_tx in node.listtransactions(): + assert logical_tx.get('address') != changeaddr + + def test_sending_from_reused_address_without_avoid_reuse(self): ''' - Test the same as test_fund_send_fund_send, except send the 10 BTC with + Test the same as test_sending_from_reused_address_fails, except send the 10 BTC with the avoid_reuse flag set to false. This means the 10 BTC send should succeed, - where it fails in test_fund_send_fund_send. + where it fails in test_sending_from_reused_address_fails. ''' - self.log.info("Test fund send fund send dirty") + self.log.info("Test sending from reused address with avoid_reuse=false") fundaddr = self.nodes[1].getnewaddress() retaddr = self.nodes[0].getnewaddress() @@ -188,7 +217,7 @@ class AvoidReuseTest(BitcoinTestFramework): assert_approx(self.nodes[1].getbalance(), 5, 0.001) assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 5, 0.001) - def test_fund_send_fund_send(self, second_addr_type): + def test_sending_from_reused_address_fails(self, second_addr_type): ''' Test the simple case where [1] generates a new address A, then [0] sends 10 BTC to A. @@ -197,7 +226,7 @@ class AvoidReuseTest(BitcoinTestFramework): [1] tries to spend 10 BTC (fails; dirty). [1] tries to spend 4 BTC (succeeds; change address sufficient) ''' - self.log.info("Test fund send fund send") + self.log.info("Test sending from reused {} address fails".format(second_addr_type)) fundaddr = self.nodes[1].getnewaddress(label="", address_type="legacy") retaddr = self.nodes[0].getnewaddress() @@ -288,5 +317,66 @@ class AvoidReuseTest(BitcoinTestFramework): assert_unspent(self.nodes[1], total_count=2, total_sum=6, reused_count=1, reused_sum=1) assert_balances(self.nodes[1], mine={"used": 1, "trusted": 5}) + def test_full_destination_group_is_preferred(self): + ''' + Test the case where [1] only has 11 outputs of 1 BTC in the same reused + address and tries to send a small payment of 0.5 BTC. The wallet + should use 10 outputs from the reused address as inputs and not a + single 1 BTC input, in order to join several outputs from the reused + address. + ''' + self.log.info("Test that full destination groups are preferred in coin selection") + + # Node under test should be empty + assert_equal(self.nodes[1].getbalance(avoid_reuse=False), 0) + + new_addr = self.nodes[1].getnewaddress() + ret_addr = self.nodes[0].getnewaddress() + + # Send 11 outputs of 1 BTC to the same, reused address in the wallet + for _ in range(11): + self.nodes[0].sendtoaddress(new_addr, 1) + + self.nodes[0].generate(1) + self.sync_all() + + # Sending a transaction that is smaller than each one of the + # available outputs + txid = self.nodes[1].sendtoaddress(address=ret_addr, amount=0.5) + inputs = self.nodes[1].getrawtransaction(txid, 1)["vin"] + + # The transaction should use 10 inputs exactly + assert_equal(len(inputs), 10) + + def test_all_destination_groups_are_used(self): + ''' + Test the case where [1] only has 22 outputs of 1 BTC in the same reused + address and tries to send a payment of 20.5 BTC. The wallet + should use all 22 outputs from the reused address as inputs. + ''' + self.log.info("Test that all destination groups are used") + + # Node under test should be empty + assert_equal(self.nodes[1].getbalance(avoid_reuse=False), 0) + + new_addr = self.nodes[1].getnewaddress() + ret_addr = self.nodes[0].getnewaddress() + + # Send 22 outputs of 1 BTC to the same, reused address in the wallet + for _ in range(22): + self.nodes[0].sendtoaddress(new_addr, 1) + + self.nodes[0].generate(1) + self.sync_all() + + # Sending a transaction that needs to use the full groups + # of 10 inputs but also the incomplete group of 2 inputs. + txid = self.nodes[1].sendtoaddress(address=ret_addr, amount=20.5) + inputs = self.nodes[1].getrawtransaction(txid, 1)["vin"] + + # The transaction should use 22 inputs exactly + assert_equal(len(inputs), 22) + + if __name__ == '__main__': AvoidReuseTest().main() diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index fb80a06433..9dd91b2495 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet backup features. diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index a5f9a047ed..8efa66a856 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet balance RPC methods.""" @@ -107,7 +107,7 @@ class WalletTest(BitcoinTestFramework): # First argument of getbalance must be set to "*" assert_raises_rpc_error(-32, "dummy first argument must be excluded or set to \"*\"", self.nodes[1].getbalance, "") - self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs") + self.log.info("Test balances with unconfirmed inputs") # Before `test_balance()`, we have had two nodes with a balance of 50 # each and then we: @@ -148,6 +148,18 @@ class WalletTest(BitcoinTestFramework): def test_balances(*, fee_node_1=0): + # getbalances + expected_balances_0 = {'mine': {'immature': Decimal('0E-8'), + 'trusted': Decimal('9.99'), # change from node 0's send + 'untrusted_pending': Decimal('60.0')}, + 'watchonly': {'immature': Decimal('5000'), + 'trusted': Decimal('50.0'), + 'untrusted_pending': Decimal('0E-8')}} + expected_balances_1 = {'mine': {'immature': Decimal('0E-8'), + 'trusted': Decimal('0E-8'), # node 1's send had an unsafe input + 'untrusted_pending': Decimal('30.0') - fee_node_1}} # Doesn't include output of node 0's send since it was spent + assert_equal(self.nodes[0].getbalances(), expected_balances_0) + assert_equal(self.nodes[1].getbalances(), expected_balances_1) # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send assert_equal(self.nodes[1].getbalance(), Decimal('0')) # node 1's send had an unsafe input @@ -160,11 +172,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0')) # getunconfirmedbalance assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) # output of node 1's spend - assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('60')) - assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) - assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('30') - fee_node_1) # Doesn't include output of node 0's send since it was spent - assert_equal(self.nodes[1].getbalances()['mine']['untrusted_pending'], Decimal('30') - fee_node_1) + # getwalletinfo.unconfirmed_balance + assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('30') - fee_node_1) test_balances(fee_node_1=Decimal('0.01')) @@ -174,15 +184,19 @@ class WalletTest(BitcoinTestFramework): self.nodes[0].sendrawtransaction(txs[1]['hex']) # sending on both nodes is faster than waiting for propagation self.sync_all() - self.log.info("Test getbalance and getunconfirmedbalance with conflicted unconfirmed inputs") + self.log.info("Test getbalance and getbalances.mine.untrusted_pending with conflicted unconfirmed inputs") test_balances(fee_node_1=Decimal('0.02')) self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY) self.sync_all() # balances are correct after the transactions are confirmed - assert_equal(self.nodes[0].getbalance(), Decimal('69.99')) # node 1's send plus change from node 0's send - assert_equal(self.nodes[1].getbalance(), Decimal('29.98')) # change from node 0's send + balance_node0 = Decimal('69.99') # node 1's send plus change from node 0's send + balance_node1 = Decimal('29.98') # change from node 0's send + assert_equal(self.nodes[0].getbalances()['mine']['trusted'], balance_node0) + assert_equal(self.nodes[1].getbalances()['mine']['trusted'], balance_node1) + assert_equal(self.nodes[0].getbalance(), balance_node0) + assert_equal(self.nodes[1].getbalance(), balance_node1) # Send total balance away from node 1 txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')]) @@ -200,13 +214,13 @@ class WalletTest(BitcoinTestFramework): # check mempool transactions count for wallet unconfirmed balance after # dynamically loading the wallet. - before = self.nodes[1].getunconfirmedbalance() + before = self.nodes[1].getbalances()['mine']['untrusted_pending'] dst = self.nodes[1].getnewaddress() self.nodes[1].unloadwallet('') self.nodes[0].sendtoaddress(dst, 0.1) self.sync_all() self.nodes[1].loadwallet('') - after = self.nodes[1].getunconfirmedbalance() + after = self.nodes[1].getbalances()['mine']['untrusted_pending'] assert_equal(before + Decimal('0.1'), after) # Create 3 more wallet txs, where the last is not accepted to the diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 38c9807757..c09ca8854f 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the bumpfee RPC. @@ -82,7 +82,6 @@ class BumpFeeTest(BitcoinTestFramework): test_notmine_bumpfee_fails(self, rbf_node, peer_node, dest_address) test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_address) test_dust_to_fee(self, rbf_node, dest_address) - test_settxfee(self, rbf_node, dest_address) test_watchonly_psbt(self, peer_node, rbf_node, dest_address) test_rebumping(self, rbf_node, dest_address) test_rebumping_not_replaceable(self, rbf_node, dest_address) @@ -90,6 +89,7 @@ class BumpFeeTest(BitcoinTestFramework): test_bumpfee_metadata(self, rbf_node, dest_address) test_locked_wallet_fails(self, rbf_node, dest_address) test_change_script_match(self, rbf_node, dest_address) + test_settxfee(self, rbf_node, dest_address) test_maxtxfee_fails(self, rbf_node, dest_address) # These tests wipe out a number of utxos that are expected in other tests test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) @@ -253,12 +253,17 @@ def test_dust_to_fee(self, rbf_node, dest_address): self.log.info('Test that bumped output that is dust is dropped to fee') rbfid = spend_one_input(rbf_node, dest_address) fulltx = rbf_node.getrawtransaction(rbfid, 1) - # size of transaction (p2wpkh, 1 input, 2 outputs): 141 vbytes - assert_equal(fulltx["vsize"], 141) - # bump with fee_rate of 0.00350000 BTC per 1000 vbytes - # expected bump fee of 141 vbytes * fee_rate 0.00350000 BTC / 1000 vbytes = 0.00049350 BTC - # but dust is dropped, so actual bump fee is 0.00050000 - bumped_tx = rbf_node.bumpfee(rbfid, {"fee_rate": 0.0035}) + # The DER formatting used by Bitcoin to serialize ECDSA signatures means that signatures can have a + # variable size of 70-72 bytes (or possibly even less), with most being 71 or 72 bytes. The signature + # in the witness is divided by 4 for the vsize, so this variance can take the weight across a 4-byte + # boundary. Thus expected transaction size (p2wpkh, 1 input, 2 outputs) is 140-141 vbytes, usually 141. + if not 140 <= fulltx["vsize"] <= 141: + raise AssertionError("Invalid tx vsize of {} (140-141 expected), full tx: {}".format(fulltx["vsize"], fulltx)) + # Bump with fee_rate of 0.00350250 BTC per 1000 vbytes to create dust. + # Expected fee is 141 vbytes * fee_rate 0.00350250 BTC / 1000 vbytes = 0.00049385 BTC. + # or occasionally 140 vbytes * fee_rate 0.00350250 BTC / 1000 vbytes = 0.00049035 BTC. + # Dust should be dropped to the fee, so actual bump fee is 0.00050000 BTC. + bumped_tx = rbf_node.bumpfee(rbfid, {"fee_rate": 0.00350250}) full_bumped_tx = rbf_node.getrawtransaction(bumped_tx["txid"], 1) assert_equal(bumped_tx["fee"], Decimal("0.00050000")) assert_equal(len(fulltx["vout"]), 2) @@ -281,9 +286,15 @@ def test_settxfee(self, rbf_node, dest_address): assert_greater_than(Decimal("0.00001000"), abs(requested_feerate - actual_feerate)) rbf_node.settxfee(Decimal("0.00000000")) # unset paytxfee + # check that settxfee respects -maxtxfee + self.restart_node(1, ['-maxtxfee=0.000025'] + self.extra_args[1]) + assert_raises_rpc_error(-8, "txfee cannot be more than wallet max tx fee", rbf_node.settxfee, Decimal('0.00003')) + self.restart_node(1, self.extra_args[1]) + rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + def test_maxtxfee_fails(self, rbf_node, dest_address): - self.log.info('Test that bumpfee fails when it hits -matxfee') + self.log.info('Test that bumpfee fails when it hits -maxtxfee') # size of bumped transaction (p2wpkh, 1 input, 2 outputs): 141 vbytes # expected bump fee of 141 vbytes * 0.00200000 BTC / 1000 vbytes = 0.00002820 BTC # which exceeds maxtxfee and is expected to raise diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index b24d312e27..48c0fcb731 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test createwallet arguments. diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index a39dfc7895..cc349c7bc5 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" +import datetime import os +import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -18,6 +20,7 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): Also check that the old hd_master is inactive """ with open(file_name, encoding='utf8') as inputfile: + found_comments = [] found_legacy_addr = 0 found_p2sh_segwit_addr = 0 found_bech32_addr = 0 @@ -26,8 +29,12 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): found_addr_rsv = 0 hd_master_addr_ret = None for line in inputfile: - # only read non comment lines - if line[0] != "#" and len(line) > 10: + line = line.strip() + if not line: + continue + if line[0] == '#': + found_comments.append(line) + else: # split out some data key_date_label, comment = line.split("#") key_date_label = key_date_label.split(" ") @@ -82,7 +89,7 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): found_script_addr += 1 break - return found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret + return found_comments, found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret class WalletDumpTest(BitcoinTestFramework): @@ -122,12 +129,36 @@ class WalletDumpTest(BitcoinTestFramework): # its capacity self.nodes[0].keypoolrefill() - # dump unencrypted wallet + self.log.info('Mine a block one second before the wallet is dumped') + dump_time = int(time.time()) + self.nodes[0].setmocktime(dump_time - 1) + self.nodes[0].generate(1) + self.nodes[0].setmocktime(dump_time) + dump_time_str = '# * Created on {}Z'.format( + datetime.datetime.fromtimestamp( + dump_time, + tz=datetime.timezone.utc, + ).replace(tzinfo=None).isoformat()) + dump_best_block_1 = '# * Best block at time of backup was {} ({}),'.format( + self.nodes[0].getblockcount(), + self.nodes[0].getbestblockhash(), + ) + dump_best_block_2 = '# mined on {}Z'.format( + datetime.datetime.fromtimestamp( + dump_time - 1, + tz=datetime.timezone.utc, + ).replace(tzinfo=None).isoformat()) + + self.log.info('Dump unencrypted wallet') result = self.nodes[0].dumpwallet(wallet_unenc_dump) assert_equal(result['filename'], wallet_unenc_dump) - found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ + found_comments, found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ read_dump(wallet_unenc_dump, addrs, [multisig_addr], None) + assert '# End of dump' in found_comments # Check that file is not corrupt + assert_equal(dump_time_str, next(c for c in found_comments if c.startswith('# * Created on'))) + assert_equal(dump_best_block_1, next(c for c in found_comments if c.startswith('# * Best block'))) + assert_equal(dump_best_block_2, next(c for c in found_comments if c.startswith('# mined on'))) assert_equal(found_legacy_addr, test_addr_count) # all keys must be in the dump assert_equal(found_p2sh_segwit_addr, test_addr_count) # all keys must be in the dump assert_equal(found_bech32_addr, test_addr_count) # all keys must be in the dump @@ -142,8 +173,12 @@ class WalletDumpTest(BitcoinTestFramework): self.nodes[0].keypoolrefill() self.nodes[0].dumpwallet(wallet_enc_dump) - found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, _ = \ + found_comments, found_legacy_addr, found_p2sh_segwit_addr, found_bech32_addr, found_script_addr, found_addr_chg, found_addr_rsv, _ = \ read_dump(wallet_enc_dump, addrs, [multisig_addr], hd_master_addr_unenc) + assert '# End of dump' in found_comments # Check that file is not corrupt + assert_equal(dump_time_str, next(c for c in found_comments if c.startswith('# * Created on'))) + assert_equal(dump_best_block_1, next(c for c in found_comments if c.startswith('# * Best block'))) + assert_equal(dump_best_block_2, next(c for c in found_comments if c.startswith('# mined on'))) assert_equal(found_legacy_addr, test_addr_count) # all keys must be in the dump assert_equal(found_p2sh_segwit_addr, test_addr_count) # all keys must be in the dump assert_equal(found_bech32_addr, test_addr_count) # all keys must be in the dump diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index 261a43472b..9dd55b4ab1 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet group functionality.""" diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 7497475b67..e4328f2b0e 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Hierarchical Deterministic wallet function.""" diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index b8b85b7a19..4ff7f1d525 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -40,7 +40,6 @@ Rescan = enum.Enum("Rescan", "no yes late_timestamp") class Variant(collections.namedtuple("Variant", "call data address_type rescan prune")): """Helper for importing one key and verifying scanned transactions.""" - def do_import(self, timestamp): """Call one key import RPC.""" rescan = self.rescan == Rescan.yes @@ -146,6 +145,7 @@ class ImportRescanTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + len(IMPORT_NODES) self.supports_cli = False + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -226,5 +226,6 @@ class ImportRescanTest(BitcoinTestFramework): variant.expected_txs += 1 variant.check(variant.sent_txid, variant.sent_amount, variant.confirmation_height) + if __name__ == "__main__": ImportRescanTest().main() diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index e3aeb61197..9e2f00e62f 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -5,6 +5,7 @@ """Test the wallet keypool and interaction with wallet encryption/locking.""" import time +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error @@ -53,12 +54,12 @@ class KeyPoolTest(BitcoinTestFramework): assert_raises_rpc_error(-12, "Keypool ran out", nodes[0].getrawchangeaddress) # drain the external keys - addr.add(nodes[0].getnewaddress()) - addr.add(nodes[0].getnewaddress()) - addr.add(nodes[0].getnewaddress()) - addr.add(nodes[0].getnewaddress()) - addr.add(nodes[0].getnewaddress()) - addr.add(nodes[0].getnewaddress()) + addr.add(nodes[0].getnewaddress(address_type="bech32")) + addr.add(nodes[0].getnewaddress(address_type="bech32")) + addr.add(nodes[0].getnewaddress(address_type="bech32")) + addr.add(nodes[0].getnewaddress(address_type="bech32")) + addr.add(nodes[0].getnewaddress(address_type="bech32")) + addr.add(nodes[0].getnewaddress(address_type="bech32")) assert len(addr) == 6 # the next one should fail assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) @@ -82,5 +83,52 @@ class KeyPoolTest(BitcoinTestFramework): assert_equal(wi['keypoolsize_hd_internal'], 100) assert_equal(wi['keypoolsize'], 100) + # create a blank wallet + nodes[0].createwallet(wallet_name='w2', blank=True) + w2 = nodes[0].get_wallet_rpc('w2') + + # refer to initial wallet as w1 + w1 = nodes[0].get_wallet_rpc('') + + # import private key and fund it + address = addr.pop() + privkey = w1.dumpprivkey(address) + res = w2.importmulti([{'scriptPubKey': {'address': address}, 'keys': [privkey], 'timestamp': 'now'}]) + assert_equal(res[0]['success'], True) + w1.walletpassphrase('test', 100) + + res = w1.sendtoaddress(address=address, amount=0.00010000) + nodes[0].generate(1) + destination = addr.pop() + + # Using a fee rate (10 sat / byte) well above the minimum relay rate + # creating a 5,000 sat transaction with change should not be possible + assert_raises_rpc_error(-4, "Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.", w2.walletcreatefundedpsbt, inputs=[], outputs=[{addr.pop(): 0.00005000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + + # creating a 10,000 sat transaction without change, with a manual input, should still be possible + res = w2.walletcreatefundedpsbt(inputs=w2.listunspent(), outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + assert_equal("psbt" in res, True) + + # creating a 10,000 sat transaction without change should still be possible + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010}) + assert_equal("psbt" in res, True) + # should work without subtractFeeFromOutputs if the exact fee is subtracted from the amount + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008900}], options={"feeRate": 0.00010}) + assert_equal("psbt" in res, True) + + # dust change should be removed + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00008800}], options={"feeRate": 0.00010}) + assert_equal("psbt" in res, True) + + # create a transaction without change at the maximum fee rate, such that the output is still spendable: + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.0008824}) + assert_equal("psbt" in res, True) + assert_equal(res["fee"], Decimal("0.00009706")) + + # creating a 10,000 sat transaction with a manual change address should be possible + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.00010, "changeAddress": addr.pop()}) + assert_equal("psbt" in res, True) + + if __name__ == '__main__': KeyPoolTest().main() diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 8c44a070b8..8ff663ccd2 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -97,6 +97,8 @@ class ListTransactionsTest(BitcoinTestFramework): txid = self.nodes[1].sendtoaddress(multisig["address"], 0.1) self.nodes[1].generate(1) self.sync_all() + assert_equal(len(self.nodes[0].listtransactions(label="watchonly", include_watchonly=True)), 1) + assert_equal(len(self.nodes[0].listtransactions(dummy="watchonly", include_watchonly=True)), 1) assert len(self.nodes[0].listtransactions(label="watchonly", count=100, include_watchonly=False)) == 0 assert_array_result(self.nodes[0].listtransactions(label="watchonly", count=100, include_watchonly=True), {"category": "receive", "amount": Decimal("0.1")}, diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index a2c502f280..c569416292 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiwallet. Verify that a bitcoind node can load multiple wallet files """ +from decimal import Decimal import os import shutil import time @@ -125,10 +126,6 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(['-salvagewallet', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file") self.nodes[0].assert_start_raises_init_error(['-salvagewallet=1', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file") - self.log.info("Do not allow -upgradewallet with multiwallet") - self.nodes[0].assert_start_raises_init_error(['-upgradewallet', '-wallet=w1', '-wallet=w2'], "Error: -upgradewallet is only allowed with a single wallet file") - self.nodes[0].assert_start_raises_init_error(['-upgradewallet=1', '-wallet=w1', '-wallet=w2'], "Error: -upgradewallet is only allowed with a single wallet file") - # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) @@ -193,9 +190,9 @@ class MultiWalletTest(BitcoinTestFramework): self.log.info('Check for per-wallet settxfee call') assert_equal(w1.getwalletinfo()['paytxfee'], 0) assert_equal(w2.getwalletinfo()['paytxfee'], 0) - w2.settxfee(4.0) + w2.settxfee(0.001) assert_equal(w1.getwalletinfo()['paytxfee'], 0) - assert_equal(w2.getwalletinfo()['paytxfee'], 4.0) + assert_equal(w2.getwalletinfo()['paytxfee'], Decimal('0.00100000')) self.log.info("Test dynamic wallet loading") @@ -332,18 +329,5 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].unloadwallet(wallet) self.nodes[1].loadwallet(wallet) - # Fail to load if wallet is downgraded - shutil.copytree(os.path.join(self.options.data_wallets_dir, 'high_minversion'), wallet_dir('high_minversion')) - self.restart_node(0, extra_args=['-upgradewallet={}'.format(FEATURE_LATEST)]) - assert {'name': 'high_minversion'} in self.nodes[0].listwalletdir()['wallets'] - self.log.info("Fail -upgradewallet that results in downgrade") - assert_raises_rpc_error( - -4, - 'Wallet loading failed: Error loading {}: Wallet requires newer version of {}'.format( - wallet_dir('high_minversion', 'wallet.dat'), self.config['environment']['PACKAGE_NAME']), - lambda: self.nodes[0].loadwallet(filename='high_minversion'), - ) - - if __name__ == '__main__': MultiWalletTest().main() diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index d122e3db52..db5902f820 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet resends transactions periodically.""" diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index bb93060739..2455d3a3c3 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run fuzz test targets. @@ -147,6 +147,7 @@ def merge_inputs(*, corpus, test_list, build_dir, merge_dir): args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', t), '-merge=1', + '-use_value_profile=1', # Also done by oss-fuzz https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487 os.path.join(corpus, t), os.path.join(merge_dir, t), ] diff --git a/test/lint/extended-lint-all.sh b/test/lint/extended-lint-all.sh index 65c51e02f5..be5d9db4a9 100755 --- a/test/lint/extended-lint-all.sh +++ b/test/lint/extended-lint-all.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/extended-lint-cppcheck.sh b/test/lint/extended-lint-cppcheck.sh index ae18d74ebf..20021d8605 100755 --- a/test/lint/extended-lint-cppcheck.sh +++ b/test/lint/extended-lint-cppcheck.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index bbd94dd6c7..6bd02d45ac 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh index 184c3682c8..cb3ec09ae6 100755 --- a/test/lint/lint-format-strings.sh +++ b/test/lint/lint-format-strings.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index 1cece6a525..bd9c8337ac 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 70410d7405..e2bb403c4d 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh index 773855bed1..7257919c98 100755 --- a/test/lint/lint-python-utf8-encoding.sh +++ b/test/lint/lint-python-utf8-encoding.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-shebang.sh b/test/lint/lint-shebang.sh index a666fdfecf..a5c8aa42b2 100755 --- a/test/lint/lint-shebang.sh +++ b/test/lint/lint-shebang.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index f59b2c9945..5540a0f74f 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # |