diff options
116 files changed, 2178 insertions, 546 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 9464ec1685..517cd93585 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -45,8 +45,6 @@ task: folder: "/tmp/ccache_dir" depends_built_cache: folder: "/tmp/cirrus-ci-build/depends/built" - depends_sdk_cache: - folder: "/tmp/cirrus-ci-build/depends/sdk-sources" install_script: - apt-get update - apt-get -y install git bash ccache diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.md b/.github/ISSUE_TEMPLATE/good_first_issue.md new file mode 100644 index 0000000000..6782218616 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/good_first_issue.md @@ -0,0 +1,20 @@ +--- +name: Good first issue +about: '(Regular devs only): Suggest a new good first issue' +title: '' +labels: good first issue +assignees: '' + +--- + +The purpose of the `good first issue` label is to highlight which issues are suitable for a new contributor without a deep understanding of the codebase. + +Useful skills: + +(For example, “C++11 std::thread”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”.) + +Want to work on this issue? + +You do not need to request permission to start working on this. You are encouraged to comment on the issue if you are planning to work on it. This will help other contributors monitor which issues are actively being addressed and is also an effective way to request assistance if and when you need it. + +For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request. diff --git a/.travis.yml b/.travis.yml index 3ddafda6d2..51bb7d6e0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,9 +94,11 @@ jobs: - set -o errexit; source ./ci/extended_lint/06_script.sh - stage: test - name: 'ARM [GOAL: install] [unit tests, no functional tests]' + name: 'ARM [GOAL: install] [unit tests, functional tests]' + arch: arm64 env: >- FILE_ENV="./ci/test/00_setup_env_arm.sh" + QEMU_USER_CMD="" # Can run the tests natively without qemu - stage: test name: 'Win64 [GOAL: deploy] [unit tests, no gui, no functional tests]' @@ -12,7 +12,7 @@ with no central authority: managing transactions and issuing money are carried out collectively by the network. Bitcoin Core is the name of open source software which enables the use of this currency. -For more information, as well as an immediately useable, binary version of +For more information, as well as an immediately usable, binary version of the Bitcoin Core software, see https://bitcoincore.org/en/download/, or read the [original whitepaper](https://bitcoincore.org/bitcoin.pdf). diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index d579dc2ed5..83d054af5f 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -116,8 +116,10 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ if test "x$bitcoin_cv_static_qt" = xyes; then _BITCOIN_QT_FIND_STATIC_PLUGINS AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static]) - _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) - AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + if test "x$TARGET_OS" != xandroid; then + _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) + AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + fi if test "x$TARGET_OS" = xwindows; then _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QWindowsIntegrationPlugin)],[-lqwindows]) AC_DEFINE(QT_QPA_PLATFORM_WINDOWS, 1, [Define this symbol if the qt platform is windows]) @@ -128,6 +130,9 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AX_CHECK_LINK_FLAG([[-framework IOKit]],[QT_LIBS="$QT_LIBS -framework IOKit"],[AC_MSG_ERROR(could not iokit framework)]) _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin)],[-lqcocoa]) AC_DEFINE(QT_QPA_PLATFORM_COCOA, 1, [Define this symbol if the qt platform is cocoa]) + elif test "x$TARGET_OS" = xandroid; then + QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype -lQt5EglSupport $QT_LIBS" + AC_DEFINE(QT_QPA_PLATFORM_ANDROID, 1, [Define this symbol if the qt platform is android]) fi fi CPPFLAGS=$TEMP_CPPFLAGS @@ -345,6 +350,9 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ if test -d "$qt_plugin_path/accessible"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/accessible" fi + if test -d "$qt_plugin_path/platforms/android"; then + QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL" + fi if test "x$use_pkgconfig" = xyes; then : dnl m4_ifdef([PKG_CHECK_MODULES],[ diff --git a/ci/README.md b/ci/README.md index 16c481158f..fb1cd7460b 100644 --- a/ci/README.md +++ b/ci/README.md @@ -12,7 +12,7 @@ To allow for a wide range of tested environments, but also ensure reproducibilit requires `docker` to be installed. To install all requirements on Ubuntu, run ``` -sudo apt install docker.io ccache bash git +sudo apt install docker.io bash git ``` To run the default test stage, diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 9335f0b337..6e2542584c 100644 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -7,11 +7,16 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf -export QEMU_USER_CMD="qemu-arm -L /usr/arm-linux-gnueabihf/" -export PACKAGES="python3 g++-arm-linux-gnueabihf busybox qemu-user" +# The host arch is unknown, so we run the tests through qemu. +# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. +export QEMU_USER_CMD="${QEMU_USER_CMD:"qemu-arm -L /usr/arm-linux-gnueabihf/"}" +# We don't know whether the host can run the cross compiled binaries. To run them, either qemu-user or libc6:armhf for +# the target is required, so install both. +export DPKG_ADD_ARCH="armhf" +export PACKAGES="python3 g++-arm-linux-gnueabihf busybox qemu-user libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true -export RUN_FUNCTIONAL_TESTS=false +export RUN_FUNCTIONAL_TESTS=true export GOAL="install" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index d0831a4c13..271ae82e5c 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -33,7 +33,7 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then fi mkdir -p "${BASE_SCRATCH_DIR}" -ccache echo "Creating ccache dir if it didn't already exist" +mkdir -p "${CCACHE_DIR}" if [ ! -d ${DIR_QA_ASSETS} ]; then git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} @@ -76,6 +76,9 @@ else DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) fi +if [ -n "$DPKG_ADD_ARCH" ]; then + DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" +fi if [ "$TRAVIS_OS_NAME" != "osx" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get update diff --git a/configure.ac b/configure.ac index 9f2942dc9f..8bde6d9bc4 100644 --- a/configure.ac +++ b/configure.ac @@ -597,6 +597,7 @@ case $host in ;; *android*) dnl make sure android stays above linux for hosts like *linux-android* + TARGET_OS=android LEVELDB_TARGET_FLAGS="-DOS_ANDROID" ;; *linux*) diff --git a/contrib/devtools/utxo_snapshot.sh b/contrib/devtools/utxo_snapshot.sh new file mode 100755 index 0000000000..dee25ff67b --- /dev/null +++ b/contrib/devtools/utxo_snapshot.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +export LC_ALL=C + +set -ueo pipefail + +if (( $# < 3 )); then + echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <bitcoin-cli-call ...>' + echo + echo " if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the " + echo " expected assumeutxo hash" + echo + echo 'Examples:' + echo + echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata" + echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli' + exit 1 +fi + +GENERATE_AT_HEIGHT="${1}"; shift; +OUTPUT_PATH="${1}"; shift; +# Most of the calls we make take a while to run, so pad with a lengthy timeout. +BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999" + +# Block we'll invalidate/reconsider to rewind/fast-forward the chain. +PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) ) + +(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while") +${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}" + +if [[ "${OUTPUT_PATH}" = "-" ]]; then + (>&2 echo "Generating txoutset info...") + ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g' +else + (>&2 echo "Generating UTXO snapshot...") + ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}" +fi + +(>&2 echo "Restoring chain to original height; this may take a while") +${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index ca11d7a0f0..23b656cad7 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -106,7 +106,7 @@ chain for " target " development.")) (base-libc glibc-2.27) (base-gcc (make-gcc-rpath-link (make-ssp-fixed-gcc gcc-9)))) - "Convienience wrapper around MAKE-CROSS-TOOLCHAIN with default values + "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target base-gcc-for-libc diff --git a/depends/Makefile b/depends/Makefile index 80df0e46f8..cec99777ff 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -58,6 +58,11 @@ full_host_os:=$(subst $(host_arch)-$(host_vendor)-,,$(canonical_host)) host_os:=$(findstring linux,$(full_host_os)) host_os+=$(findstring darwin,$(full_host_os)) host_os+=$(findstring mingw32,$(full_host_os)) + +ifeq (android,$(findstring android,$(full_host_os))) +host_os:=android +endif + host_os:=$(strip $(host_os)) ifeq ($(host_os),) host_os=$(full_host_os) diff --git a/depends/README.md b/depends/README.md index ca542be13f..e5e2a8a653 100644 --- a/depends/README.md +++ b/depends/README.md @@ -30,8 +30,12 @@ Common `host-platform-triplets` for cross compilation are: - `aarch64-linux-gnu` for Linux ARM 64 bit - `riscv32-linux-gnu` for Linux RISC-V 32 bit - `riscv64-linux-gnu` for Linux RISC-V 64 bit +- `armv7a-linux-android` for Android ARM 32 bit +- `aarch64-linux-android` for Android ARM 64 bit +- `i686-linux-android` for Android x86 32 bit +- `x86_64-linux-android` for Android x86 64 bit -No other options are needed, the paths are automatically configured. +The paths are automatically configured and no other options are needed unless targeting [Android](#Android). ### Install the required dependencies: Ubuntu & Debian @@ -90,6 +94,19 @@ options will be passed to bitcoin's configure. In this case, `--disable-wallet`. download-win: run 'make download-win' to fetch all sources needed for win builds download-linux: run 'make download-linux' to fetch all sources needed for linux builds + +### Android + +Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). +In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. + +API levels from 24 to 29 have been tested to work. + +If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. +This is an example command for a default build with no disabled dependencies: + + ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin + ### Other documentation - [description.md](description.md): General description of the depends system diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk new file mode 100644 index 0000000000..969ec2a1cb --- /dev/null +++ b/depends/hosts/android.mk @@ -0,0 +1,11 @@ +ifeq ($(HOST),armv7a-linux-android) +android_AR=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ar +android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang++ +android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang +android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ranlib +else +android_AR=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ar +android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ +android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang +android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib +endif diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 766b8d224e..d360bb5ba6 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -15,12 +15,17 @@ $(package)_config_opts_mingw32=binary-format=pe target-os=windows threadapi=win3 $(package)_config_opts_x86_64_mingw32=address-model=64 $(package)_config_opts_i686_mingw32=address-model=32 $(package)_config_opts_i686_linux=address-model=32 architecture=x86 +$(package)_config_opts_i686_android=address-model=32 +$(package)_config_opts_aarch64_android=address-model=64 +$(package)_config_opts_x86_64_android=address-model=64 +$(package)_config_opts_armv7a_android=address-model=32 $(package)_toolset_$(host_os)=gcc $(package)_archiver_$(host_os)=$($(package)_ar) $(package)_toolset_darwin=clang-darwin $(package)_config_libraries=chrono,filesystem,system,thread,test $(package)_cxxflags=-std=c++11 -fvisibility=hidden $(package)_cxxflags_linux=-fPIC +$(package)_cxxflags_android=-fPIC endef define $(package)_preprocess_cmds diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 300c806a8d..b4333abaf0 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -3,16 +3,24 @@ $(package)_version=2.1.8-stable $(package)_download_path=https://github.com/libevent/libevent/archive/ $(package)_file_name=release-$($(package)_version).tar.gz $(package)_sha256_hash=316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d +$(package)_patches=fix_android_arc4random_addrandom.patch -define $(package)_preprocess_cmds - ./autogen.sh -endef +ifneq (,$(findstring android,$(host))) + define $(package)_preprocess_cmds + ./autogen.sh && patch -p1 < $($(package)_patch_dir)/fix_android_arc4random_addrandom.patch + endef +else + define $(package)_preprocess_cmds + ./autogen.sh + endef +endif define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic + $(package)_config_opts_android=--with-pic endef define $(package)_config_cmds diff --git a/depends/packages/openssl.mk b/depends/packages/openssl.mk index 3498a3f6ee..e3b3647dd9 100644 --- a/depends/packages/openssl.mk +++ b/depends/packages/openssl.mk @@ -58,6 +58,11 @@ $(package)_config_opts_riscv64_linux=linux-generic64 $(package)_config_opts_x86_64_darwin=darwin64-x86_64-cc $(package)_config_opts_x86_64_mingw32=mingw64 $(package)_config_opts_i686_mingw32=mingw +$(package)_config_opts_android=-fPIC +$(package)_config_opts_aarch64_android=linux-generic64 +$(package)_config_opts_x86_64_android=linux-generic64 +$(package)_config_opts_armv7a_android=linux-generic32 +$(package)_config_opts_i686_android=linux-generic32 endef define $(package)_preprocess_cmds diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 35f8b829db..09734c7e6f 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -3,6 +3,7 @@ packages:=boost openssl libevent qt_packages = qrencode zlib qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig +qt_android_packages=qt rapidcheck_packages = rapidcheck diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index 21570726ff..d1687883bc 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -9,6 +9,7 @@ $(package)_config_opts=--disable-shared --without-tools --without-tests --disabl $(package)_config_opts += --disable-gprof --disable-gcov --disable-mudflap $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_linux=--with-pic +$(package)_config_opts_android=--with-pic endef define $(package)_preprocess_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 0685eb2da2..a0f7cda2c0 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -8,7 +8,7 @@ $(package)_dependencies=zlib $(package)_linux_dependencies=freetype fontconfig libxcb $(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib -$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch +$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch fix_android_qmake_conf.patch fix_android_jni_static.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=fb5a47799754af73d3bf501fe513342cfe2fc37f64e80df5533f6110e804220c @@ -128,6 +128,26 @@ $(package)_config_opts_x86_64_linux = -xplatform linux-g++-64 $(package)_config_opts_aarch64_linux = -xplatform linux-aarch64-gnu-g++ $(package)_config_opts_riscv64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ $(package)_config_opts_mingw32 = -no-opengl -xplatform win32-g++ -device-option CROSS_COMPILE="$(host)-" + +$(package)_config_opts_android = -xplatform android-clang +$(package)_config_opts_android += -android-sdk $(ANDROID_SDK) +$(package)_config_opts_android += -android-ndk $(ANDROID_NDK) +$(package)_config_opts_android += -android-ndk-platform android-$(ANDROID_API_LEVEL) +$(package)_config_opts_android += -device-option CROSS_COMPILE="$(host)-" +$(package)_config_opts_android += -egl +$(package)_config_opts_android += -qpa xcb +$(package)_config_opts_android += -no-eglfs +$(package)_config_opts_android += -opengl es2 +$(package)_config_opts_android += -qt-freetype +$(package)_config_opts_android += -no-fontconfig +$(package)_config_opts_android += -L $(host_prefix)/lib +$(package)_config_opts_android += -I $(host_prefix)/include + +$(package)_config_opts_aarch64_android += -android-arch arm64-v8a +$(package)_config_opts_armv7a_android += -android-arch armeabi-v7a +$(package)_config_opts_x86_64_android += -android-arch x86_64 +$(package)_config_opts_i686_android += -android-arch i686 + $(package)_build_env = QT_RCC_TEST=1 $(package)_build_env += QT_RCC_SOURCE_DATE_OVERRIDE=1 endef @@ -170,6 +190,8 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch &&\ patch -p1 -i $($(package)_patch_dir)/fix_rcc_determinism.patch &&\ patch -p1 -i $($(package)_patch_dir)/xkb-default.patch &&\ + patch -p1 -i $($(package)_patch_dir)/fix_android_qmake_conf.patch &&\ + patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch &&\ echo "!host_build: QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index ce85c437bb..6f35ede248 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -11,6 +11,7 @@ define $(package)_set_vars $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking $(package)_config_opts_linux=--with-pic + $(package)_config_opts_android=--with-pic $(package)_cxxflags=-std=c++11 endef diff --git a/depends/packages/zlib.mk b/depends/packages/zlib.mk index 168f85e65e..acb02020a8 100644 --- a/depends/packages/zlib.mk +++ b/depends/packages/zlib.mk @@ -11,6 +11,7 @@ $(package)_config_opts+=RANLIB="$($(package)_ranlib)" $(package)_config_opts+=AR="$($(package)_ar)" $(package)_config_opts_darwin+=AR="$($(package)_libtool)" $(package)_config_opts_darwin+=ARFLAGS="-o" +$(package)_config_opts_android+=CHOST=$(host) endef # zlib has its own custom configure script that takes in options like CC, diff --git a/depends/patches/libevent/fix_android_arc4random_addrandom.patch b/depends/patches/libevent/fix_android_arc4random_addrandom.patch new file mode 100644 index 0000000000..5bcc64bef6 --- /dev/null +++ b/depends/patches/libevent/fix_android_arc4random_addrandom.patch @@ -0,0 +1,68 @@ +From cadae3ab7abf45e61ecae8aac39d97d1f3cbd336 Mon Sep 17 00:00:00 2001 +From: Lawrence Nahum <lawrence@greenaddress.it> +Date: Sun, 3 Dec 2017 22:56:09 +0100 +Subject: [PATCH] fixup + +--- + configure.ac | 1 + + evutil_rand.c | 3 +++ + include/event2/util.h | 4 ++-- + 3 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/configure.ac b/configure.ac +index 7528d37..3bb2121 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -341,6 +341,7 @@ dnl Checks for library functions. + AC_CHECK_FUNCS([ \ + accept4 \ + arc4random \ ++ arc4random_addrandom \ + arc4random_buf \ + eventfd \ + epoll_create1 \ +diff --git a/evutil_rand.c b/evutil_rand.c +index 046a14b..3f0bf2c 100644 +--- a/evutil_rand.c ++++ b/evutil_rand.c +@@ -191,6 +191,7 @@ evutil_secure_rng_get_bytes(void *buf, size_t n) + { + ev_arc4random_buf(buf, n); + } ++#ifdef HAVE_ARC4RANDOM_ADDRANDOM + + void + evutil_secure_rng_add_bytes(const char *buf, size_t n) +@@ -199,6 +200,8 @@ evutil_secure_rng_add_bytes(const char *buf, size_t n) + n>(size_t)INT_MAX ? INT_MAX : (int)n); + } + ++#endif ++ + void + evutil_free_secure_rng_globals_(void) + { +diff --git a/include/event2/util.h b/include/event2/util.h +index dd4bbb6..a9a169d 100644 +--- a/include/event2/util.h ++++ b/include/event2/util.h +@@ -841,7 +841,7 @@ int evutil_secure_rng_init(void); + */ + EVENT2_EXPORT_SYMBOL + int evutil_secure_rng_set_urandom_device_file(char *fname); +- ++#ifdef HAVE_ARC4RANDOM_ADDRANDOM + /** Seed the random number generator with extra random bytes. + + You should almost never need to call this function; it should be +@@ -858,7 +858,7 @@ int evutil_secure_rng_set_urandom_device_file(char *fname); + */ + EVENT2_EXPORT_SYMBOL + void evutil_secure_rng_add_bytes(const char *dat, size_t datlen); +- ++#endif + #ifdef __cplusplus + } + #endif +-- +2.14.3 diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch new file mode 100644 index 0000000000..2f6ff00f40 --- /dev/null +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -0,0 +1,18 @@ +--- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp ++++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp +@@ -890,6 +890,14 @@ + __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); + return -1; + } ++ ++ const jint ret = QT_PREPEND_NAMESPACE(QtAndroidPrivate::initJNI(vm, env)); ++ if (ret != 0) ++ { ++ __android_log_print(ANDROID_LOG_FATAL, "Qt", "initJNI failed"); ++ return ret; ++ } ++ + QWindowSystemInterfacePrivate::TabletEvent::setPlatformSynthesizesMouse(false); + + m_javaVM = vm; + diff --git a/depends/patches/qt/fix_android_qmake_conf.patch b/depends/patches/qt/fix_android_qmake_conf.patch new file mode 100644 index 0000000000..13bfff9776 --- /dev/null +++ b/depends/patches/qt/fix_android_qmake_conf.patch @@ -0,0 +1,20 @@ +--- old/qtbase/mkspecs/android-clang/qmake.conf ++++ new/qtbase/mkspecs/android-clang/qmake.conf +@@ -30,7 +30,7 @@ + QMAKE_CFLAGS += -target mips64el-none-linux-android + + QMAKE_CFLAGS += -gcc-toolchain $$NDK_TOOLCHAIN_PATH +-QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a ++QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a -nostdlib++ + QMAKE_CFLAGS += -DANDROID_HAS_WSTRING --sysroot=$$NDK_ROOT/sysroot \ + -isystem $$NDK_ROOT/sysroot/usr/include/$$NDK_TOOLS_PREFIX \ + -isystem $$NDK_ROOT/sources/cxx-stl/llvm-libc++/include \ +@@ -40,7 +40,7 @@ + ANDROID_SOURCES_CXX_STL_LIBDIR = $$NDK_ROOT/sources/cxx-stl/llvm-libc++/libs/$$ANDROID_TARGET_ARCH + + ANDROID_STDCPP_PATH = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++_shared.so +-ANDROID_CXX_STL_LIBS = -lc++ ++ANDROID_CXX_STL_LIBS = -lc++_shared + + QMAKE_ARM_CFLAGS_RELEASE = -Oz + QMAKE_ARM_CFLAGS_RELEASE_WITH_DEBUGINFO = -g -Oz diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 3b8b0db162..a63391e01a 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -50,7 +50,7 @@ Given a height: returns hash of block in best-block-chain at height provided. Returns various state info regarding block chain processing. Only supports JSON as output format. -* chain : (string) current network name as defined in BIP70 (main, test, regtest) +* chain : (string) current network name (main, test, regtest) * blocks : (numeric) the current number of blocks processed in the server * headers : (numeric) the current number of headers we have validated * bestblockhash : (string) the hash of the currently best block diff --git a/doc/bips.md b/doc/bips.md index 45562cec62..b96862297f 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -19,7 +19,11 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.19.0**): * [`BIP 65`](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki): The CHECKLOCKTIMEVERIFY softfork was merged in **v0.12.0** ([PR #6351](https://github.com/bitcoin/bitcoin/pull/6351)), and backported to **v0.11.2** and **v0.10.4**. Mempool-only CLTV was added in [PR #6124](https://github.com/bitcoin/bitcoin/pull/6124). * [`BIP 66`](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki): The strict DER rules and associated version 3 blocks have been implemented since **v0.10.0** ([PR #5713](https://github.com/bitcoin/bitcoin/pull/5713)). * [`BIP 68`](https://github.com/bitcoin/bips/blob/master/bip-0068.mediawiki): Sequence locks have been implemented as of **v0.12.1** ([PR #7184](https://github.com/bitcoin/bitcoin/pull/7184)), and have been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). -* [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). Support can be optionally disabled at build time since **v0.18.0** ([PR 14451](https://github.com/bitcoin/bitcoin/pull/14451)), and is disabled by default at build time since **v0.19.0** ([PR #15584](https://github.com/bitcoin/bitcoin/pull/15584)). +* [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): + Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). + Support can be optionally disabled at build time since **v0.18.0** ([PR 14451](https://github.com/bitcoin/bitcoin/pull/14451)), + and it is disabled by default at build time since **v0.19.0** ([PR #15584](https://github.com/bitcoin/bitcoin/pull/15584)). + It has been removed as of **v0.20.0** ([PR 17165](https://github.com/bitcoin/bitcoin/pull/17165)). * [`BIP 90`](https://github.com/bitcoin/bips/blob/master/bip-0090.mediawiki): Trigger mechanism for activation of BIPs 34, 65, and 66 has been simplified to block height checks since **v0.14.0** ([PR #8391](https://github.com/bitcoin/bitcoin/pull/8391)). * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 1a7ce91ca6..e7fd8102a4 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -384,7 +384,7 @@ Threads - ThreadScriptCheck : Verifies block scripts. -- ThreadImport : Loads blocks from blk*.dat files or bootstrap.dat. +- ThreadImport : Loads blocks from `blk*.dat` files or `-loadblock=<file>`. - StartNode : Starts other threads. @@ -654,7 +654,7 @@ Strings and formatting - Do not use it to convert to `QString`. Use `QString::fromStdString()`. - - *Rationale*: Qt has build-in functionality for converting their string + - *Rationale*: Qt has built-in functionality for converting their string type from/to C++. No need to roll your own. - In cases where do you call `.c_str()`, you might want to additionally check that the string does not contain embedded '\0' characters, because diff --git a/doc/release-notes-15954.md b/doc/release-notes-15954.md new file mode 100644 index 0000000000..f4d2c5688c --- /dev/null +++ b/doc/release-notes-15954.md @@ -0,0 +1,4 @@ +Configuration option changes +----------------------------- + +Importing blocks upon startup via the `bootstrap.dat` file no longer occurs by default. The file must now be specified with `-loadblock=<file>`. diff --git a/doc/release-notes.md b/doc/release-notes.md index ea82962e75..a47c8802b0 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -85,6 +85,7 @@ Wallet ------ - The wallet now by default uses bech32 addresses when using RPC, and creates native segwit change outputs. +- The way that output trust was computed has been fixed in #16766, which impacts confirmed/unconfirmed balance status and coin selection. Low-level changes ================= diff --git a/src/Makefile.am b/src/Makefile.am index 619f968bc9..ff4f071a3c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -146,12 +146,14 @@ BITCOIN_CORE_H = \ dbwrapper.h \ limitedmap.h \ logging.h \ + logging/timer.h \ memusage.h \ merkleblock.h \ miner.h \ net.h \ net_permissions.h \ net_processing.h \ + net_types.h \ netaddress.h \ netbase.h \ netmessagemaker.h \ @@ -160,6 +162,7 @@ BITCOIN_CORE_H = \ node/context.h \ node/psbt.h \ node/transaction.h \ + node/utxo_snapshot.h \ noui.h \ optional.h \ outputtype.h \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 38143e32b9..fbcab86d8f 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -30,6 +30,7 @@ bench_bench_bitcoin_SOURCES = \ bench/gcs_filter.cpp \ bench/merkle_root.cpp \ bench/mempool_eviction.cpp \ + bench/mempool_stress.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ bench/util_time.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index c3f0120005..742530c55c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -7,6 +7,7 @@ FUZZ_TARGETS = \ test/fuzz/address_deserialize \ test/fuzz/addrman_deserialize \ test/fuzz/banentry_deserialize \ + test/fuzz/bech32 \ test/fuzz/block_deserialize \ test/fuzz/blockheader_deserialize \ test/fuzz/blocklocator_deserialize \ @@ -56,18 +57,26 @@ RAW_TEST_FILES = GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) BITCOIN_TEST_SUITE = \ - test/lib/transaction_utils.h \ + test/lib/blockfilter.cpp \ + test/lib/blockfilter.h \ + test/lib/logging.cpp \ + test/lib/logging.h \ test/lib/transaction_utils.cpp \ + test/lib/transaction_utils.h \ test/main.cpp \ test/setup_common.h \ - test/setup_common.cpp + test/setup_common.cpp \ + test/util/str.h \ + test/util/str.cpp FUZZ_SUITE = \ - test/setup_common.h \ - test/setup_common.cpp \ test/fuzz/fuzz.cpp \ test/fuzz/fuzz.h \ - test/fuzz/FuzzedDataProvider.h + test/fuzz/FuzzedDataProvider.h \ + test/setup_common.cpp \ + test/setup_common.h \ + test/util/str.cpp \ + test/util/str.h FUZZ_SUITE_LD_COMMON = \ $(LIBBITCOIN_SERVER) \ @@ -118,6 +127,7 @@ BITCOIN_TESTS =\ test/key_io_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ + test/logging_tests.cpp \ test/dbwrapper_tests.cpp \ test/validation_tests.cpp \ test/mempool_tests.cpp \ @@ -240,6 +250,12 @@ test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_bech32_SOURCES = $(FUZZ_SUITE) test/fuzz/bech32.cpp +test_fuzz_bech32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_bech32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_bech32_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_bech32_LDADD = $(FUZZ_SUITE_LD_COMMON) + test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) diff --git a/src/addrdb.h b/src/addrdb.h index 290b63dd12..ad85224d1f 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -7,6 +7,7 @@ #define BITCOIN_ADDRDB_H #include <fs.h> +#include <net_types.h> // For banmap_t #include <serialize.h> #include <string> @@ -79,8 +80,6 @@ public: } }; -typedef std::map<CSubNet, CBanEntry> banmap_t; - /** Access to the (IP) address database (peers.dat) */ class CAddrDB { diff --git a/src/banman.h b/src/banman.h index 9d45bf0559..7943f666e8 100644 --- a/src/banman.h +++ b/src/banman.h @@ -10,6 +10,7 @@ #include <addrdb.h> #include <fs.h> +#include <net_types.h> // For banmap_t #include <sync.h> // NOTE: When adjusting this, update rpcnet:setban's help ("24h") diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index f2b520e893..1b6b1736a9 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -112,7 +112,7 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double printer.header(); for (const auto& p : benchmarks()) { - TestingSetup test{CBaseChainParams::REGTEST}; + RegTestingSetup test{}; { LOCK(cs_main); assert(::ChainActive().Height() == 0); diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp new file mode 100644 index 0000000000..389e2c096f --- /dev/null +++ b/src/bench/mempool_stress.cpp @@ -0,0 +1,87 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <policy/policy.h> +#include <txmempool.h> + +#include <vector> + +static void AddTx(const CTransactionRef& tx, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) +{ + int64_t nTime = 0; + unsigned int nHeight = 1; + bool spendsCoinbase = false; + unsigned int sigOpCost = 4; + LockPoints lp; + pool.addUnchecked(CTxMemPoolEntry(tx, 1000, nTime, nHeight, spendsCoinbase, sigOpCost, lp)); +} + +struct Available { + CTransactionRef ref; + size_t vin_left{0}; + size_t tx_count; + Available(CTransactionRef& ref, size_t tx_count) : ref(ref), tx_count(tx_count){} +}; + +static void ComplexMemPool(benchmark::State& state) +{ + FastRandomContext det_rand{true}; + std::vector<Available> available_coins; + std::vector<CTransactionRef> ordered_coins; + // Create some base transactions + size_t tx_counter = 1; + for (auto x = 0; x < 100; ++x) { + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(1); + tx.vin[0].scriptSig = CScript() << CScriptNum(tx_counter); + tx.vin[0].scriptWitness.stack.push_back(CScriptNum(x).getvch()); + tx.vout.resize(det_rand.randrange(10)+2); + for (auto& out : tx.vout) { + out.scriptPubKey = CScript() << CScriptNum(tx_counter) << OP_EQUAL; + out.nValue = 10 * COIN; + } + ordered_coins.emplace_back(MakeTransactionRef(tx)); + available_coins.emplace_back(ordered_coins.back(), tx_counter++); + } + for (auto x = 0; x < 800 && !available_coins.empty(); ++x) { + CMutableTransaction tx = CMutableTransaction(); + size_t n_ancestors = det_rand.randrange(10)+1; + for (size_t ancestor = 0; ancestor < n_ancestors && !available_coins.empty(); ++ancestor){ + size_t idx = det_rand.randrange(available_coins.size()); + Available coin = available_coins[idx]; + uint256 hash = coin.ref->GetHash(); + // biased towards taking just one ancestor, but maybe more + size_t n_to_take = det_rand.randrange(2) == 0 ? 1 : 1+det_rand.randrange(coin.ref->vout.size() - coin.vin_left); + for (size_t i = 0; i < n_to_take; ++i) { + tx.vin.emplace_back(); + tx.vin.back().prevout = COutPoint(hash, coin.vin_left++); + tx.vin.back().scriptSig = CScript() << coin.tx_count; + tx.vin.back().scriptWitness.stack.push_back(CScriptNum(coin.tx_count).getvch()); + } + if (coin.vin_left == coin.ref->vin.size()) { + coin = available_coins.back(); + available_coins.pop_back(); + } + tx.vout.resize(det_rand.randrange(10)+2); + for (auto& out : tx.vout) { + out.scriptPubKey = CScript() << CScriptNum(tx_counter) << OP_EQUAL; + out.nValue = 10 * COIN; + } + } + ordered_coins.emplace_back(MakeTransactionRef(tx)); + available_coins.emplace_back(ordered_coins.back(), tx_counter++); + } + CTxMemPool pool; + LOCK2(cs_main, pool.cs); + while (state.KeepRunning()) { + for (auto& tx : ordered_coins) { + AddTx(tx, pool); + } + pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); + pool.TrimToSize(GetVirtualTransactionSize(*ordered_coins.front())); + } +} + +BENCHMARK(ComplexMemPool, 1); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index d7b6891503..592fcbe8dd 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -265,7 +265,7 @@ public: result.pushKV("proxy", batch[ID_NETWORKINFO]["result"]["networks"][0]["proxy"]); result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); - if (!batch[ID_WALLETINFO].isNull()) { + if (!batch[ID_WALLETINFO]["result"].isNull()) { result.pushKV("walletversion", batch[ID_WALLETINFO]["result"]["walletversion"]); result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); result.pushKV("keypoololdest", batch[ID_WALLETINFO]["result"]["keypoololdest"]); diff --git a/src/bloom.h b/src/bloom.h index 7d3aa878b0..c3f64ba4bc 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -115,9 +115,6 @@ public: class CRollingBloomFilter { public: - // A random bloom filter calls GetRand() at creation time. - // Don't create global CRollingBloomFilter objects, as they may be - // constructed before the randomizer is properly initialized. CRollingBloomFilter(const unsigned int nElements, const double nFPRate); void insert(const std::vector<unsigned char>& vKey); diff --git a/src/chainparams.h b/src/chainparams.h index 8f1d27e03c..6be066806b 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -75,7 +75,7 @@ public: uint64_t AssumedChainStateSize() const { return m_assumed_chain_state_size; } /** Whether it is possible to mine blocks on demand (no retargeting) */ bool MineBlocksOnDemand() const { return consensus.fPowNoRetargeting; } - /** Return the BIP70 network string (main, test or regtest) */ + /** Return the network string */ std::string NetworkIDString() const { return strNetworkID; } /** Return the list of hostnames to look up for DNS seeds */ const std::vector<std::string>& DNSSeeds() const { return vSeeds; } @@ -120,7 +120,7 @@ std::unique_ptr<const CChainParams> CreateChainParams(const std::string& chain); const CChainParams &Params(); /** - * Sets the params returned by Params() to those for the given BIP70 chain name. + * Sets the params returned by Params() to those for the given chain name. * @throws std::runtime_error when the chain is not supported. */ void SelectParams(const std::string& chain); diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index f34646f7ac..69fe2438f3 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -15,10 +15,12 @@ class CBaseChainParams { public: - /** BIP70 chain name strings (main, test or regtest) */ + ///@{ + /** Chain name strings */ static const std::string MAIN; static const std::string TESTNET; static const std::string REGTEST; + ///@} const std::string& DataDir() const { return strDataDir; } int RPCPort() const { return nRPCPort; } diff --git a/src/init.cpp b/src/init.cpp index 1a99ca9abc..3eb8fc7976 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -372,7 +372,7 @@ void SetupServerArgs() gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-loadblock=<file>", "Imports blocks from external blk000??.dat file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -684,20 +684,6 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) LoadGenesisBlock(chainparams); } - // hardcoded $DATADIR/bootstrap.dat - fs::path pathBootstrap = GetDataDir() / "bootstrap.dat"; - if (fs::exists(pathBootstrap)) { - FILE *file = fsbridge::fopen(pathBootstrap, "rb"); - if (file) { - fs::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; - LogPrintf("Importing bootstrap.dat...\n"); - LoadExternalBlockFile(chainparams, file); - RenameOver(pathBootstrap, pathBootstrapOld); - } else { - LogPrintf("Warning: Could not open bootstrap file %s\n", pathBootstrap.string()); - } - } - // -loadblock= for (const fs::path& path : vImportFiles) { FILE *file = fsbridge::fopen(path, "rb"); diff --git a/src/interfaces/node.h b/src/interfaces/node.h index c29037f2e3..adf3de7b07 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -5,9 +5,9 @@ #ifndef BITCOIN_INTERFACES_NODE_H #define BITCOIN_INTERFACES_NODE_H -#include <addrdb.h> // For banmap_t #include <amount.h> // For CAmount #include <net.h> // For CConnman::NumConnections +#include <net_types.h> // For banmap_t #include <netaddress.h> // For Network #include <support/allocators/secure.h> // For SecureString diff --git a/src/logging.cpp b/src/logging.cpp index 60ab486198..b01177f23f 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -67,6 +67,9 @@ bool BCLog::Logger::StartLogging() if (m_print_to_file) FileWriteStr(s, m_fileout); if (m_print_to_console) fwrite(s.data(), 1, s.size(), stdout); + for (const auto& cb : m_print_callbacks) { + cb(s); + } m_msgs_before_open.pop_front(); } @@ -81,6 +84,7 @@ void BCLog::Logger::DisconnectTestLogger() m_buffering = true; if (m_fileout != nullptr) fclose(m_fileout); m_fileout = nullptr; + m_print_callbacks.clear(); } void BCLog::Logger::EnableCategory(BCLog::LogFlags flag) @@ -270,6 +274,9 @@ void BCLog::Logger::LogPrintStr(const std::string& str) fwrite(str_prefixed.data(), 1, str_prefixed.size(), stdout); fflush(stdout); } + for (const auto& cb : m_print_callbacks) { + cb(str_prefixed); + } if (m_print_to_file) { assert(m_fileout != nullptr); diff --git a/src/logging.h b/src/logging.h index e37c0c823b..9ed41c2b98 100644 --- a/src/logging.h +++ b/src/logging.h @@ -77,6 +77,9 @@ namespace BCLog { std::string LogTimestampStr(const std::string& str); + /** Slots that connect to the print signal */ + std::list<std::function<void(const std::string&)>> m_print_callbacks /* GUARDED_BY(m_cs) */ {}; + public: bool m_print_to_console = false; bool m_print_to_file = false; @@ -95,7 +98,22 @@ namespace BCLog { bool Enabled() const { std::lock_guard<std::mutex> scoped_lock(m_cs); - return m_buffering || m_print_to_console || m_print_to_file; + return m_buffering || m_print_to_console || m_print_to_file || !m_print_callbacks.empty(); + } + + /** Connect a slot to the print signal and return the connection */ + std::list<std::function<void(const std::string&)>>::iterator PushBackCallback(std::function<void(const std::string&)> fun) + { + std::lock_guard<std::mutex> scoped_lock(m_cs); + m_print_callbacks.push_back(std::move(fun)); + return --m_print_callbacks.end(); + } + + /** Delete a connection */ + void DeleteCallback(std::list<std::function<void(const std::string&)>>::iterator it) + { + std::lock_guard<std::mutex> scoped_lock(m_cs); + m_print_callbacks.erase(it); } /** Start logging (and flush all buffered messages) */ diff --git a/src/logging/timer.h b/src/logging/timer.h new file mode 100644 index 0000000000..34dbb942c5 --- /dev/null +++ b/src/logging/timer.h @@ -0,0 +1,104 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_LOGGING_TIMER_H +#define BITCOIN_LOGGING_TIMER_H + +#include <logging.h> +#include <util/macros.h> +#include <util/time.h> + +#include <chrono> +#include <string> + + +namespace BCLog { + +//! RAII-style object that outputs timing information to logs. +template <typename TimeType> +class Timer +{ +public: + //! If log_category is left as the default, end_msg will log unconditionally + //! (instead of being filtered by category). + Timer( + std::string prefix, + std::string end_msg, + BCLog::LogFlags log_category = BCLog::LogFlags::ALL) : + m_prefix(std::move(prefix)), + m_title(std::move(end_msg)), + m_log_category(log_category) + { + this->Log(strprintf("%s started", m_title)); + m_start_t = GetTime<std::chrono::microseconds>(); + } + + ~Timer() + { + this->Log(strprintf("%s completed", m_title)); + } + + void Log(const std::string& msg) + { + const std::string full_msg = this->LogMsg(msg); + + if (m_log_category == BCLog::LogFlags::ALL) { + LogPrintf("%s\n", full_msg); + } else { + LogPrint(m_log_category, "%s\n", full_msg); + } + } + + std::string LogMsg(const std::string& msg) + { + const auto end_time = GetTime<std::chrono::microseconds>() - m_start_t; + if (m_start_t.count() <= 0) { + return strprintf("%s: %s", m_prefix, msg); + } + + std::string units = ""; + float divisor = 1; + + if (std::is_same<TimeType, std::chrono::microseconds>::value) { + units = "μs"; + } else if (std::is_same<TimeType, std::chrono::milliseconds>::value) { + units = "ms"; + divisor = 1000.; + } else if (std::is_same<TimeType, std::chrono::seconds>::value) { + units = "s"; + divisor = 1000. * 1000.; + } + + const float time_ms = end_time.count() / divisor; + return strprintf("%s: %s (%.2f%s)", m_prefix, msg, time_ms, units); + } + +private: + std::chrono::microseconds m_start_t{}; + + //! Log prefix; usually the name of the function this was created in. + const std::string m_prefix{}; + + //! A descriptive message of what is being timed. + const std::string m_title{}; + + //! Forwarded on to LogPrint if specified - has the effect of only + //! outputing the timing log when a particular debug= category is specified. + const BCLog::LogFlags m_log_category{}; + +}; + +} // namespace BCLog + + +#define LOG_TIME_MICROS(end_msg, ...) \ + BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) +#define LOG_TIME_MILLIS(end_msg, ...) \ + BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) +#define LOG_TIME_SECONDS(end_msg, ...) \ + BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) + + +#endif // BITCOIN_LOGGING_TIMER_H diff --git a/src/net.cpp b/src/net.cpp index 674f2ecf24..84692d2a79 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2666,11 +2666,10 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn addrBind(addrBindIn), fInbound(fInboundIn), nKeyedNetGroup(nKeyedNetGroupIn), - addrKnown(5000, 0.001), // Don't relay addr messages to peers that we connect to as block-relay-only // peers (to prevent adversaries from inferring these links from addr // traffic). - m_addr_relay_peer(!block_relay_only), + m_addr_known{block_relay_only ? nullptr : MakeUnique<CRollingBloomFilter>(5000, 0.001)}, id(idIn), nLocalHostNonce(nLocalHostNonceIn), nLocalServices(nLocalServicesIn), @@ -776,13 +776,12 @@ public: // flood relay std::vector<CAddress> vAddrToSend; - CRollingBloomFilter addrKnown; + const std::unique_ptr<CRollingBloomFilter> m_addr_known; bool fGetAddr{false}; int64_t nNextAddrSend GUARDED_BY(cs_sendProcessing){0}; int64_t nNextLocalAddrSend GUARDED_BY(cs_sendProcessing){0}; - const bool m_addr_relay_peer; - bool IsAddrRelayPeer() const { return m_addr_relay_peer; } + bool IsAddrRelayPeer() const { return m_addr_known != nullptr; } // List of block ids we still have announce. // There is no final sorting before sending, as they are always sent immediately @@ -809,7 +808,7 @@ public: bool fSendMempool GUARDED_BY(cs_tx_inventory){false}; // Last time a "MEMPOOL" request was serviced. std::atomic<std::chrono::seconds> m_last_mempool_req{std::chrono::seconds{0}}; - int64_t nNextInvSend{0}; + std::chrono::microseconds nNextInvSend{0}; CCriticalSection cs_feeFilter; // Minimum fee rate with which to filter inv's to this node @@ -931,7 +930,8 @@ public: void AddAddressKnown(const CAddress& _addr) { - addrKnown.insert(_addr.GetKey()); + assert(m_addr_known); + m_addr_known->insert(_addr.GetKey()); } void PushAddress(const CAddress& _addr, FastRandomContext &insecure_rand) @@ -939,7 +939,8 @@ public: // Known checking here is only to save space from duplicates. // SendMessages will filter it again for knowns that were added // after addresses were pushed. - if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) { + assert(m_addr_known); + if (_addr.IsValid() && !m_addr_known->contains(_addr.GetKey())) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; } else { @@ -990,11 +991,13 @@ public: void MaybeSetAddrName(const std::string& addrNameIn); }; - - - - /** Return a timestamp in the future (in microseconds) for exponentially distributed events. */ int64_t PoissonNextSend(int64_t now, int average_interval_seconds); +/** Wrapper to return mockable type */ +inline std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, std::chrono::seconds average_interval) +{ + return std::chrono::microseconds{PoissonNextSend(now.count(), average_interval.count())}; +} + #endif // BITCOIN_NET_H diff --git a/src/net_processing.cpp b/src/net_processing.cpp index d03817834d..f42a26ca3e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1340,7 +1340,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connma // Relay to a limited number of other nodes // Use deterministic randomness to send to the same nodes for 24 hours - // at a time so the addrKnowns of the chosen nodes prevent repeats + // at a time so the m_addr_knowns of the chosen nodes prevent repeats uint64_t hashAddr = addr.GetHash(); const CSipHasher hasher = connman->GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); FastRandomContext insecure_rand; @@ -3575,6 +3575,8 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Address refresh broadcast int64_t nNow = GetTimeMicros(); + auto current_time = GetTime<std::chrono::microseconds>(); + if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { AdvertiseLocal(pto); pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); @@ -3587,11 +3589,12 @@ bool PeerLogicValidation::SendMessages(CNode* pto) pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); std::vector<CAddress> vAddr; vAddr.reserve(pto->vAddrToSend.size()); + assert(pto->m_addr_known); for (const CAddress& addr : pto->vAddrToSend) { - if (!pto->addrKnown.contains(addr.GetKey())) + if (!pto->m_addr_known->contains(addr.GetKey())) { - pto->addrKnown.insert(addr.GetKey()); + pto->m_addr_known->insert(addr.GetKey()); vAddr.push_back(addr); // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) @@ -3795,13 +3798,13 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LOCK(pto->m_tx_relay->cs_tx_inventory); // Check whether periodic sends should happen bool fSendTrickle = pto->HasPermission(PF_NOBAN); - if (pto->m_tx_relay->nNextInvSend < nNow) { + if (pto->m_tx_relay->nNextInvSend < current_time) { fSendTrickle = true; if (pto->fInbound) { - pto->m_tx_relay->nNextInvSend = connman->PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL); + pto->m_tx_relay->nNextInvSend = std::chrono::microseconds{connman->PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL)}; } else { // Use half the delay for outbound peers, as there is less privacy concern for them. - pto->m_tx_relay->nNextInvSend = PoissonNextSend(nNow, INVENTORY_BROADCAST_INTERVAL >> 1); + pto->m_tx_relay->nNextInvSend = PoissonNextSend(current_time, std::chrono::seconds{INVENTORY_BROADCAST_INTERVAL >> 1}); } } @@ -3916,7 +3919,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling - const auto current_time = GetTime<std::chrono::microseconds>(); + current_time = GetTime<std::chrono::microseconds>(); // nNow is the current system time (GetTimeMicros is not mockable) and // should be replaced by the mockable current_time eventually nNow = GetTimeMicros(); diff --git a/src/net_types.h b/src/net_types.h new file mode 100644 index 0000000000..d55a8cde6c --- /dev/null +++ b/src/net_types.h @@ -0,0 +1,15 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NET_TYPES_H +#define BITCOIN_NET_TYPES_H + +#include <map> + +class CBanEntry; +class CSubNet; + +using banmap_t = std::map<CSubNet, CBanEntry>; + +#endif // BITCOIN_NET_TYPES_H diff --git a/src/netbase.cpp b/src/netbase.cpp index 0148aea428..d1cde8c40f 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -605,7 +605,7 @@ static void LogConnectFailure(bool manual_connection, const char* fmt, const Arg * @param nTimeout Wait this many milliseconds for the connection to be * established. * @param manual_connection Whether or not the connection was manually requested - * (e.g. thru the addnode RPC) + * (e.g. through the addnode RPC) * * @returns Whether or not a connection was successfully made. */ @@ -709,7 +709,7 @@ bool GetProxy(enum Network net, proxyType &proxyInfoOut) { /** * Set the name proxy to use for all connections to nodes specified by a - * hostname. After setting this proxy, connecting to a node sepcified by a + * hostname. After setting this proxy, connecting to a node specified by a * hostname won't result in a local lookup of said hostname, rather, connect to * the node by asking the name proxy for a proxy connection to the hostname, * effectively delegating the hostname lookup to the specified proxy. diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 57fa158ad2..a818f06d51 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -14,9 +14,6 @@ #include <map> -#include <boost/thread.hpp> - - static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); @@ -38,6 +35,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, //! Calculate statistics about the unspent transaction output set bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { + stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); assert(pcursor); @@ -51,7 +49,6 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { - boost::this_thread::interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { @@ -61,6 +58,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) } prevkey = key.hash; outputs[key.n] = std::move(coin); + stats.coins_count++; } else { return error("%s: unable to read value", __func__); } diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 7c11aab8bd..a19af0fd1b 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -15,16 +15,17 @@ class CCoinsView; struct CCoinsStats { - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nBogoSize; - uint256 hashSerialized; - uint64_t nDiskSize; - CAmount nTotalAmount; - - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} + int nHeight{0}; + uint256 hashBlock{}; + uint64_t nTransactions{0}; + uint64_t nTransactionOutputs{0}; + uint64_t nBogoSize{0}; + uint256 hashSerialized{}; + uint64_t nDiskSize{0}; + CAmount nTotalAmount{0}; + + //! The number of coins contained. + uint64_t coins_count{0}; }; //! Calculate statistics about the unspent transaction output set diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 2da3ecd8e3..3c0df2b26e 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -31,7 +31,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t CCoinsViewCache &view = ::ChainstateActive().CoinsTip(); for (size_t o = 0; o < tx->vout.size(); o++) { const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); - // IsSpent doesnt mean the coin is spent, it means the output doesnt' exist. + // IsSpent doesn't mean the coin is spent, it means the output doesn't exist. // So if the output does exist, then this transaction exists in the chain. if (!existingCoin.IsSpent()) return TransactionError::ALREADY_IN_CHAIN; } diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h new file mode 100644 index 0000000000..702a0cbe53 --- /dev/null +++ b/src/node/utxo_snapshot.h @@ -0,0 +1,50 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H +#define BITCOIN_NODE_UTXO_SNAPSHOT_H + +#include <uint256.h> +#include <serialize.h> + +//! Metadata describing a serialized version of a UTXO set from which an +//! assumeutxo CChainState can be constructed. +class SnapshotMetadata +{ +public: + //! The hash of the block that reflects the tip of the chain for the + //! UTXO set contained in this snapshot. + uint256 m_base_blockhash; + + //! The number of coins in the UTXO set contained in this snapshot. Used + //! during snapshot load to estimate progress of UTXO set reconstruction. + uint64_t m_coins_count = 0; + + //! Necessary to "fake" the base nChainTx so that we can estimate progress during + //! initial block download for the assumeutxo chainstate. + unsigned int m_nchaintx = 0; + + SnapshotMetadata() { } + SnapshotMetadata( + const uint256& base_blockhash, + uint64_t coins_count, + unsigned int nchaintx) : + m_base_blockhash(base_blockhash), + m_coins_count(coins_count), + m_nchaintx(nchaintx) { } + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(m_base_blockhash); + READWRITE(m_coins_count); + READWRITE(m_nchaintx); + } + +}; + +#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/noui.cpp b/src/noui.cpp index 11c8f1e13d..a5b7a2d591 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -66,28 +66,31 @@ void noui_connect() noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessage); } -bool noui_ThreadSafeMessageBoxSuppressed(const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeMessageBoxRedirect(const std::string& message, const std::string& caption, unsigned int style) { + LogPrintf("%s: %s\n", caption, message); return false; } -bool noui_ThreadSafeQuestionSuppressed(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeQuestionRedirect(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) { + LogPrintf("%s: %s\n", caption, message); return false; } -void noui_InitMessageSuppressed(const std::string& message) +void noui_InitMessageRedirect(const std::string& message) { + LogPrintf("init message: %s\n", message); } -void noui_suppress() +void noui_test_redirect() { noui_ThreadSafeMessageBoxConn.disconnect(); noui_ThreadSafeQuestionConn.disconnect(); noui_InitMessageConn.disconnect(); - noui_ThreadSafeMessageBoxConn = uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBoxSuppressed); - noui_ThreadSafeQuestionConn = uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestionSuppressed); - noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessageSuppressed); + noui_ThreadSafeMessageBoxConn = uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBoxRedirect); + noui_ThreadSafeQuestionConn = uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestionRedirect); + noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessageRedirect); } void noui_reconnect() diff --git a/src/noui.h b/src/noui.h index 854aeeacca..621e9c2798 100644 --- a/src/noui.h +++ b/src/noui.h @@ -17,10 +17,10 @@ void noui_InitMessage(const std::string& message); /** Connect all bitcoind signal handlers */ void noui_connect(); -/** Suppress all bitcoind signal handlers. Used to suppress output during test runs that produce expected errors */ -void noui_suppress(); +/** Redirect all bitcoind signal handlers to LogPrintf. Used to check or suppress output during test runs that produce expected errors */ +void noui_test_redirect(); -/** Reconnects the regular Non-GUI handlers after having used noui_suppress */ +/** Reconnects the regular Non-GUI handlers after having used noui_test_redirect */ void noui_reconnect(); #endif // BITCOIN_NOUI_H diff --git a/src/psbt.h b/src/psbt.h index 9d996171bb..6a5c468058 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -401,7 +401,6 @@ struct PartiallySignedTransaction bool AddInput(const CTxIn& txin, PSBTInput& psbtin); bool AddOutput(const CTxOut& txout, const PSBTOutput& psbtout); PartiallySignedTransaction() {} - PartiallySignedTransaction(const PartiallySignedTransaction& psbt_in) : tx(psbt_in.tx), inputs(psbt_in.inputs), outputs(psbt_in.outputs), unknown(psbt_in.unknown) {} explicit PartiallySignedTransaction(const CMutableTransaction& tx); /** * Finds the UTXO for a given input index diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index ae11b80347..48201b420e 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -4,9 +4,9 @@ #include <qt/bantablemodel.h> -#include <qt/clientmodel.h> - #include <interfaces/node.h> +#include <net_types.h> // For banmap_t +#include <qt/clientmodel.h> #include <algorithm> diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 02a2a01bdd..234d3865ab 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -309,7 +309,7 @@ void BitcoinApplication::requestShutdown() // rescanning a wallet. m_node.startShutdown(); // Unsetting the client model can cause the current thread to wait for node - // to complete an operation, like wait for a RPC execution to complate. + // to complete an operation, like wait for a RPC execution to complete. window->setClientModel(nullptr); pollShutdownTimer->stop(); diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 386d559281..7190d59240 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -797,7 +797,7 @@ <item> <widget class="QPushButton" name="buttonMinimizeFee"> <property name="toolTip"> - <string>collapse fee-settings</string> + <string>Hide transaction fee settings</string> </property> <property name="text"> <string>Hide</string> diff --git a/src/qt/networkstyle.h b/src/qt/networkstyle.h index bb12dd1b6e..1367261325 100644 --- a/src/qt/networkstyle.h +++ b/src/qt/networkstyle.h @@ -13,7 +13,7 @@ class NetworkStyle { public: - /** Get style associated with provided BIP70 network id, or 0 if not known */ + /** Get style associated with provided network id, or 0 if not known */ static const NetworkStyle* instantiate(const std::string& networkId); const QString &getAppName() const { return appName; } diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index d21dec118a..a873519a34 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -64,8 +64,8 @@ public: CAmount amount; // If from a payment request, this is used for storing the memo QString message; - // If building with BIP70 is disabled, keep the payment request around as - // serialized string to ensure load/store is lossless + // Keep the payment request around as a serialized string to ensure + // load/store is lossless. std::string sPaymentRequest; // Empty if no authentication or invalid signature/cert/etc. QString authenticatedMerchant; diff --git a/src/random.h b/src/random.h index 22801ec155..9d1f751773 100644 --- a/src/random.h +++ b/src/random.h @@ -166,6 +166,7 @@ public: /** Generate a random integer in the range [0..range). */ uint64_t randrange(uint64_t range) noexcept { + assert(range); --range; int bits = CountBits(range); while (true) { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 4ca8225392..2f4b4412f5 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -15,6 +15,7 @@ #include <hash.h> #include <index/blockfilterindex.h> #include <node/coinstats.h> +#include <node/utxo_snapshot.h> #include <policy/feerate.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -38,8 +39,6 @@ #include <univalue.h> -#include <boost/thread/thread.hpp> // boost::thread::interrupt - #include <condition_variable> #include <memory> #include <mutex> @@ -58,7 +57,7 @@ static CUpdatedBlock latestblock; */ double GetDifficulty(const CBlockIndex* blockindex) { - assert(blockindex); + CHECK_NONFATAL(blockindex); int nShift = (blockindex->nBits >> 24) & 0xff; double dDiff = @@ -957,7 +956,7 @@ static UniValue pruneblockchain(const JSONRPCRequest& request) PruneBlockFilesManual(height); const CBlockIndex* block = ::ChainActive().Tip(); - assert(block); + CHECK_NONFATAL(block); while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { block = block->pprev; } @@ -1191,7 +1190,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) {}, RPCResult{ "{\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"chain\": \"xxxx\", (string) current network name (main, test, regtest)\n" " \"blocks\": xxxxxx, (numeric) the height of the most-work fully-validated chain. The genesis block has height 0\n" " \"headers\": xxxxxx, (numeric) the current number of headers we have validated\n" " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" @@ -1252,7 +1251,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) obj.pushKV("pruned", fPruneMode); if (fPruneMode) { const CBlockIndex* block = tip; - assert(block); + CHECK_NONFATAL(block); while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { block = block->pprev; } @@ -1598,7 +1597,7 @@ static UniValue getchaintxstats(const JSONRPCRequest& request) } } - assert(pindex != nullptr); + CHECK_NONFATAL(pindex != nullptr); if (request.params[0].isNull()) { blockcount = std::max(0, std::min(blockcount, pindex->nHeight - 1)); @@ -1771,7 +1770,7 @@ static UniValue getblockstats(const JSONRPCRequest& request) } } - assert(pindex != nullptr); + CHECK_NONFATAL(pindex != nullptr); std::set<std::string> stats; if (!request.params[1].isNull()) { @@ -1871,7 +1870,7 @@ static UniValue getblockstats(const JSONRPCRequest& request) } CAmount txfee = tx_total_in - tx_total_out; - assert(MoneyRange(txfee)); + CHECK_NONFATAL(MoneyRange(txfee)); if (do_medianfee) { fee_array.push_back(txfee); } @@ -1975,7 +1974,6 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& Coin coin; if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false; if (++count % 8192 == 0) { - boost::this_thread::interruption_point(); if (should_abort) { // allow to abort the scan via the abort reference return false; @@ -2008,7 +2006,7 @@ public: explicit CoinsViewScanReserver() : m_could_reserve(false) {} bool reserve() { - assert (!m_could_reserve); + CHECK_NONFATAL(!m_could_reserve); std::lock_guard<std::mutex> lock(g_utxosetscan); if (g_scan_in_progress) { return false; @@ -2135,9 +2133,9 @@ UniValue scantxoutset(const JSONRPCRequest& request) LOCK(cs_main); ::ChainstateActive().ForceFlushStateToDisk(); pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); - assert(pcursor); + CHECK_NONFATAL(pcursor); tip = ::ChainActive().Tip(); - assert(tip); + CHECK_NONFATAL(tip); } bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins); result.pushKV("success", res); @@ -2245,6 +2243,113 @@ static UniValue getblockfilter(const JSONRPCRequest& request) return ret; } +/** + * Serialize the UTXO set to a file for loading elsewhere. + * + * @see SnapshotMetadata + */ +UniValue dumptxoutset(const JSONRPCRequest& request) +{ + RPCHelpMan{ + "dumptxoutset", + "\nWrite the serialized UTXO set to disk.\n" + "Incidentally flushes the latest coinsdb (leveldb) to disk.\n", + { + {"path", + RPCArg::Type::STR, + RPCArg::Optional::NO, + /* default_val */ "", + "path to the output file. If relative, will be prefixed by datadir."}, + }, + RPCResult{ + "{\n" + " \"coins_written\": n, (numeric) the number of coins written in the snapshot\n" + " \"base_hash\": \"...\", (string) the hash of the base of the snapshot\n" + " \"base_height\": n, (string) the height of the base of the snapshot\n" + " \"path\": \"...\" (string) the absolute path that the snapshot was written to\n" + "]\n" + }, + RPCExamples{ + HelpExampleCli("dumptxoutset", "utxo.dat") + } + }.Check(request); + + fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + // Write to a temporary path and then move into `path` on completion + // to avoid confusion due to an interruption. + fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + + if (fs::exists(path)) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + path.string() + " already exists. If you are sure this is what you want, " + "move it out of the way first"); + } + + FILE* file{fsbridge::fopen(temppath, "wb")}; + CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + std::unique_ptr<CCoinsViewCursor> pcursor; + CCoinsStats stats; + CBlockIndex* tip; + + { + // We need to lock cs_main to ensure that the coinsdb isn't written to + // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats + // based upon the coinsdb, and (iii) constructing a cursor to the + // coinsdb for use below this block. + // + // Cursors returned by leveldb iterate over snapshots, so the contents + // of the pcursor will not be affected by simultaneous writes during + // use below this block. + // + // See discussion here: + // https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369 + // + LOCK(::cs_main); + + ::ChainstateActive().ForceFlushStateToDisk(); + + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + + pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); + tip = LookupBlockIndex(stats.hashBlock); + CHECK_NONFATAL(tip); + } + + SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx}; + + afile << metadata; + + COutPoint key; + Coin coin; + unsigned int iter{0}; + + while (pcursor->Valid()) { + if (iter % 5000 == 0 && !IsRPCRunning()) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); + } + ++iter; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + afile << key; + afile << coin; + } + + pcursor->Next(); + } + + afile.fclose(); + fs::rename(temppath, path); + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_written", stats.coins_count); + result.pushKV("base_hash", tip->GetBlockHash().ToString()); + result.pushKV("base_height", tip->nHeight); + result.pushKV("path", path.string()); + return result; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -2281,6 +2386,7 @@ static const CRPCCommand commands[] = { "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} }, { "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} }, { "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} }, + { "hidden", "dumptxoutset", &dumptxoutset, {"path"} }, }; // clang-format on diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index b3158d1e0c..ab22155651 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -233,7 +233,7 @@ static UniValue getmininginfo(const JSONRPCRequest& request) " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" " \"networkhashps\": nnn, (numeric) The network hashes per second\n" " \"pooledtx\": n (numeric) The size of the mempool\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"chain\": \"xxxx\", (string) current network name (main, test, regtest)\n" " \"warnings\": \"...\" (string) any network and blockchain warnings\n" "}\n" }, @@ -555,7 +555,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) // Need to update only after we know CreateNewBlock succeeded pindexPrev = pindexPrevNew; } - assert(pindexPrev); + CHECK_NONFATAL(pindexPrev); CBlock* pblock = &pblocktemplate->block; // pointer for convenience const Consensus::Params& consensusParams = Params().GetConsensus(); @@ -597,7 +597,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) entry.pushKV("fee", pblocktemplate->vTxFees[index_in_template]); int64_t nTxSigOps = pblocktemplate->vTxSigOpsCost[index_in_template]; if (fPreSegWit) { - assert(nTxSigOps % WITNESS_SCALE_FACTOR == 0); + CHECK_NONFATAL(nTxSigOps % WITNESS_SCALE_FACTOR == 0); nTxSigOps /= WITNESS_SCALE_FACTOR; } entry.pushKV("sigops", nTxSigOps); @@ -686,9 +686,9 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) int64_t nSigOpLimit = MAX_BLOCK_SIGOPS_COST; int64_t nSizeLimit = MAX_BLOCK_SERIALIZED_SIZE; if (fPreSegWit) { - assert(nSigOpLimit % WITNESS_SCALE_FACTOR == 0); + CHECK_NONFATAL(nSigOpLimit % WITNESS_SCALE_FACTOR == 0); nSigOpLimit /= WITNESS_SCALE_FACTOR; - assert(nSizeLimit % WITNESS_SCALE_FACTOR == 0); + CHECK_NONFATAL(nSizeLimit % WITNESS_SCALE_FACTOR == 0); nSizeLimit /= WITNESS_SCALE_FACTOR; } result.pushKV("sigoplimit", nSigOpLimit); diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index f443f37c6d..f1dcc9b607 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -8,8 +8,9 @@ #include <clientversion.h> #include <core_io.h> #include <net.h> -#include <net_processing.h> #include <net_permissions.h> +#include <net_processing.h> +#include <net_types.h> // For banmap_t #include <netbase.h> #include <node/context.h> #include <policy/settings.h> diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 17380f113f..983f251d6b 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1620,7 +1620,7 @@ UniValue joinpsbts(const JSONRPCRequest& request) std::vector<int> output_indices(merged_psbt.outputs.size()); std::iota(output_indices.begin(), output_indices.end(), 0); - // Shuffle input and output indicies lists + // Shuffle input and output indices lists Shuffle(input_indices.begin(), input_indices.end(), FastRandomContext()); Shuffle(output_indices.begin(), output_indices.end(), FastRandomContext()); diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 5b92650764..1936998ff3 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -29,7 +29,7 @@ UniValue SignTransaction(CMutableTransaction& mtx, const SigningProvider* keysto * Parse a prevtxs UniValue array and get the map of coins from it * * @param prevTxs Array of previous txns outputs that tx depends on but may not yet be in the block chain - * @param keystore A pointer to the temprorary keystore if there is one + * @param keystore A pointer to the temporary keystore if there is one * @param coins Map of unspent outputs - coins in mempool and current chain UTXO set, may be extended by previous txns outputs after call */ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keystore, std::map<COutPoint, Coin>& coins); diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 653b287e97..cfa3509c65 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -428,7 +428,7 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP std::set<std::string> named_args; for (const auto& arg : m_args) { // Should have unique named arguments - assert(named_args.insert(arg.m_name).second); + CHECK_NONFATAL(named_args.insert(arg.m_name).second); } } @@ -620,11 +620,11 @@ std::string RPCArg::ToStringObj(const bool oneline) const case Type::OBJ: case Type::OBJ_USER_KEYS: // Currently unused, so avoid writing dead code - assert(false); + CHECK_NONFATAL(false); // no default case, so the compiler can warn about missing cases } - assert(false); + CHECK_NONFATAL(false); } std::string RPCArg::ToString(const bool oneline) const @@ -661,7 +661,7 @@ std::string RPCArg::ToString(const bool oneline) const // no default case, so the compiler can warn about missing cases } - assert(false); + CHECK_NONFATAL(false); } static std::pair<int64_t, int64_t> ParseRange(const UniValue& value) diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 4b27ef0ca9..13cdd6c61a 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -36,7 +36,7 @@ namespace { // xpubs use other characters too, but already have their own checksum // mechanism. // * Function names like "multi()" use other characters, but mistakes in -// these would generally result in an unparseable descriptor. +// these would generally result in an unparsable descriptor. // * A case error always counts as 1 symbol error. // * Any other 1 character substitution error counts as 1 or 2 symbol errors. // * Any 1 symbol error is always detected. diff --git a/src/test/bech32_tests.cpp b/src/test/bech32_tests.cpp index 0ba492c24e..a1dabee579 100644 --- a/src/test/bech32_tests.cpp +++ b/src/test/bech32_tests.cpp @@ -4,24 +4,12 @@ #include <bech32.h> #include <test/setup_common.h> +#include <test/util/str.h> #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(bech32_tests, BasicTestingSetup) -static bool CaseInsensitiveEqual(const std::string &s1, const std::string &s2) -{ - if (s1.size() != s2.size()) return false; - for (size_t i = 0; i < s1.size(); ++i) { - char c1 = s1[i]; - if (c1 >= 'A' && c1 <= 'Z') c1 -= ('A' - 'a'); - char c2 = s2[i]; - if (c2 >= 'A' && c2 <= 'Z') c2 -= ('A' - 'a'); - if (c1 != c2) return false; - } - return true; -} - BOOST_AUTO_TEST_CASE(bip173_testvectors_valid) { static const std::string CASES[] = { diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 5ce8e6feb0..df589b63bf 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -3,8 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <blockencodings.h> -#include <consensus/merkle.h> #include <chainparams.h> +#include <consensus/merkle.h> #include <pow.h> #include <streams.h> @@ -14,11 +14,7 @@ std::vector<std::pair<uint256, CTransactionRef>> extra_txn; -struct RegtestingSetup : public TestingSetup { - RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {} -}; - -BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegtestingSetup) +BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegTestingSetup) static CBlock BuildBlockTestCase() { CBlock block; diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 4a15bf0c77..acc6d6a21b 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -8,8 +8,9 @@ #include <index/blockfilterindex.h> #include <miner.h> #include <pow.h> -#include <test/setup_common.h> #include <script/standard.h> +#include <test/lib/blockfilter.h> +#include <test/setup_common.h> #include <util/time.h> #include <validation.h> @@ -17,23 +18,6 @@ BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) -static bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, - BlockFilter& filter) -{ - CBlock block; - if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { - return false; - } - - CBlockUndo block_undo; - if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) { - return false; - } - - filter = BlockFilter(filter_type, block, block_undo); - return true; -} - static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index, uint256& last_header) { diff --git a/src/test/fuzz/bech32.cpp b/src/test/fuzz/bech32.cpp new file mode 100644 index 0000000000..8b91f9bc96 --- /dev/null +++ b/src/test/fuzz/bech32.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bech32.h> +#include <test/fuzz/fuzz.h> +#include <test/util/str.h> +#include <util/strencodings.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <utility> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string random_string(buffer.begin(), buffer.end()); + const std::pair<std::string, std::vector<uint8_t>> r1 = bech32::Decode(random_string); + if (r1.first.empty()) { + assert(r1.second.empty()); + } else { + const std::string& hrp = r1.first; + const std::vector<uint8_t>& data = r1.second; + const std::string reencoded = bech32::Encode(hrp, data); + assert(CaseInsensitiveEqual(random_string, reencoded)); + } + + std::vector<unsigned char> input; + ConvertBits<8, 5, true>([&](unsigned char c) { input.push_back(c); }, buffer.begin(), buffer.end()); + const std::string encoded = bech32::Encode("bc", input); + assert(!encoded.empty()); + + const std::pair<std::string, std::vector<uint8_t>> r2 = bech32::Decode(encoded); + if (r2.first.empty()) { + assert(r2.second.empty()); + } else { + const std::string& hrp = r2.first; + const std::vector<uint8_t>& data = r2.second; + assert(hrp == "bc"); + assert(data == input); + } +} diff --git a/src/test/lib/blockfilter.cpp b/src/test/lib/blockfilter.cpp new file mode 100644 index 0000000000..ddcee85d7e --- /dev/null +++ b/src/test/lib/blockfilter.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/lib/blockfilter.h> + +#include <chainparams.h> +#include <validation.h> + + +bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter) +{ + CBlock block; + if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { + return false; + } + + CBlockUndo block_undo; + if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) { + return false; + } + + filter = BlockFilter(filter_type, block, block_undo); + return true; +} + diff --git a/src/test/lib/blockfilter.h b/src/test/lib/blockfilter.h new file mode 100644 index 0000000000..392dacbe80 --- /dev/null +++ b/src/test/lib/blockfilter.h @@ -0,0 +1,13 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_LIB_BLOCKFILTER_H +#define BITCOIN_TEST_LIB_BLOCKFILTER_H + +#include <blockfilter.h> +class CBlockIndex; + +bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter); + +#endif // BITCOIN_TEST_LIB_BLOCKFILTER_H diff --git a/src/test/lib/logging.cpp b/src/test/lib/logging.cpp new file mode 100644 index 0000000000..4cfebf63df --- /dev/null +++ b/src/test/lib/logging.cpp @@ -0,0 +1,32 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/lib/logging.h> + +#include <logging.h> +#include <noui.h> +#include <tinyformat.h> +#include <util/memory.h> + +#include <stdexcept> + +DebugLogHelper::DebugLogHelper(std::string message) + : m_message{std::move(message)} +{ + m_print_connection = LogInstance().PushBackCallback( + [this](const std::string& s) { + if (m_found) return; + m_found = s.find(m_message) != std::string::npos; + }); + noui_test_redirect(); +} + +void DebugLogHelper::check_found() +{ + noui_reconnect(); + LogInstance().DeleteCallback(m_print_connection); + if (!m_found) { + throw std::runtime_error(strprintf("'%s' not found in debug log\n", m_message)); + } +} diff --git a/src/test/lib/logging.h b/src/test/lib/logging.h new file mode 100644 index 0000000000..ea1b0ad6f0 --- /dev/null +++ b/src/test/lib/logging.h @@ -0,0 +1,29 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_LIB_LOGGING_H +#define BITCOIN_TEST_LIB_LOGGING_H + +#include <util/macros.h> + +#include <functional> +#include <list> +#include <string> + +class DebugLogHelper +{ + const std::string m_message; + bool m_found{false}; + std::list<std::function<void(const std::string&)>>::iterator m_print_connection; + + void check_found(); + +public: + DebugLogHelper(std::string message); + ~DebugLogHelper() { check_found(); } +}; + +#define ASSERT_DEBUG_LOG(message) DebugLogHelper PASTE2(debugloghelper, __COUNTER__)(message) + +#endif // BITCOIN_TEST_LIB_LOGGING_H diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp new file mode 100644 index 0000000000..eb1826ae8d --- /dev/null +++ b/src/test/logging_tests.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <logging.h> +#include <logging/timer.h> +#include <test/setup_common.h> + +#include <chrono> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(logging_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(logging_timer) +{ + + SetMockTime(1); + auto sec_timer = BCLog::Timer<std::chrono::seconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(sec_timer.LogMsg("test secs"), "tests: test secs (1.00s)"); + + SetMockTime(1); + auto ms_timer = BCLog::Timer<std::chrono::milliseconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(ms_timer.LogMsg("test ms"), "tests: test ms (1000.00ms)"); + + SetMockTime(1); + auto micro_timer = BCLog::Timer<std::chrono::microseconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(micro_timer.LogMsg("test micros"), "tests: test micros (1000000.00μs)"); + + SetMockTime(0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index fe5d31b7d3..6d2f5474ff 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -749,6 +749,43 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) pool.GetTransactionAncestry(ty6->GetHash(), ancestors, descendants); BOOST_CHECK_EQUAL(ancestors, 9ULL); BOOST_CHECK_EQUAL(descendants, 6ULL); + + /* Ancestors represented more than once ("diamond") */ + // + // [ta].0 <- [tb].0 -----<------- [td].0 + // | | + // \---1 <- [tc].0 --<--/ + // + CTransactionRef ta, tb, tc, td; + ta = make_tx(/* output_values */ {10 * COIN}); + tb = make_tx(/* output_values */ {5 * COIN, 3 * COIN}, /* inputs */ {ta}); + tc = make_tx(/* output_values */ {2 * COIN}, /* inputs */ {tb}, /* input_indices */ {1}); + td = make_tx(/* output_values */ {6 * COIN}, /* inputs */ {tb, tc}, /* input_indices */ {0, 0}); + pool.clear(); + pool.addUnchecked(entry.Fee(10000LL).FromTx(ta)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(tb)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(tc)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(td)); + + // Ancestors / descendants should be: + // transaction ancestors descendants + // ============ =================== =========== + // ta 1 (ta 4 (ta,tb,tc,td) + // tb 2 (ta,tb) 4 (ta,tb,tc,td) + // tc 3 (ta,tb,tc) 4 (ta,tb,tc,td) + // td 4 (ta,tb,tc,td) 4 (ta,tb,tc,td) + pool.GetTransactionAncestry(ta->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 1ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tb->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 2ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tc->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 3ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(td->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 4ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index f5f217b841..0c1452822d 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -301,5 +301,19 @@ BOOST_AUTO_TEST_CASE(LocalAddress_BasicLifecycle) BOOST_CHECK_EQUAL(IsLocal(addr), false); } +BOOST_AUTO_TEST_CASE(PoissonNextSend) +{ + g_mock_deterministic_tests = true; + + int64_t now = 5000; + int average_interval_seconds = 600; + + auto poisson = ::PoissonNextSend(now, average_interval_seconds); + std::chrono::microseconds poisson_chrono = ::PoissonNextSend(std::chrono::microseconds{now}, std::chrono::seconds{average_interval_seconds}); + + BOOST_CHECK_EQUAL(poisson, poisson_chrono.count()); + + g_mock_deterministic_tests = false; +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_p2sh_tests.cpp b/src/test/script_p2sh_tests.cpp index f451d80984..ec28d6a0ad 100644 --- a/src/test/script_p2sh_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -209,20 +209,21 @@ BOOST_AUTO_TEST_CASE(is) p2sh << OP_HASH160 << ToByteVector(dummy) << OP_EQUAL; BOOST_CHECK(p2sh.IsPayToScriptHash()); - // Not considered pay-to-script-hash if using one of the OP_PUSHDATA opcodes: std::vector<unsigned char> direct = {OP_HASH160, 20}; direct.insert(direct.end(), 20, 0); direct.push_back(OP_EQUAL); BOOST_CHECK(CScript(direct.begin(), direct.end()).IsPayToScriptHash()); + + // Not considered pay-to-script-hash if using one of the OP_PUSHDATA opcodes: std::vector<unsigned char> pushdata1 = {OP_HASH160, OP_PUSHDATA1, 20}; pushdata1.insert(pushdata1.end(), 20, 0); pushdata1.push_back(OP_EQUAL); BOOST_CHECK(!CScript(pushdata1.begin(), pushdata1.end()).IsPayToScriptHash()); - std::vector<unsigned char> pushdata2 = {OP_HASH160, 20, 0}; + std::vector<unsigned char> pushdata2 = {OP_HASH160, OP_PUSHDATA2, 20, 0}; pushdata2.insert(pushdata2.end(), 20, 0); pushdata2.push_back(OP_EQUAL); BOOST_CHECK(!CScript(pushdata2.begin(), pushdata2.end()).IsPayToScriptHash()); - std::vector<unsigned char> pushdata4 = {OP_HASH160, 20, 0, 0, 0}; + std::vector<unsigned char> pushdata4 = {OP_HASH160, OP_PUSHDATA4, 20, 0, 0, 0}; pushdata4.insert(pushdata4.end(), 20, 0); pushdata4.push_back(OP_EQUAL); BOOST_CHECK(!CScript(pushdata4.begin(), pushdata4.end()).IsPayToScriptHash()); diff --git a/src/test/setup_common.cpp b/src/test/setup_common.cpp index 3425bd59c1..797b72ff59 100644 --- a/src/test/setup_common.cpp +++ b/src/test/setup_common.cpp @@ -124,11 +124,12 @@ TestingSetup::~TestingSetup() pblocktree.reset(); } -TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) +TestChain100Setup::TestChain100Setup() { // CreateAndProcessBlock() does not support building SegWit blocks, so don't activate in these tests. // TODO: fix the code to support SegWit blocks. gArgs.ForceSetArg("-segwitheight", "432"); + // Need to recreate chainparams SelectParams(CBaseChainParams::REGTEST); // Generate a 100-block chain: @@ -142,12 +143,9 @@ TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) } } -// // Create a new block with just given transactions, coinbase paying to // scriptPubKey, and try to add it to the current chain. -// -CBlock -TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) +CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) { const CChainParams& chainparams = Params(); std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); @@ -175,6 +173,7 @@ TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& TestChain100Setup::~TestChain100Setup() { + gArgs.ForceSetArg("-segwitheight", "0"); } diff --git a/src/test/setup_common.h b/src/test/setup_common.h index 5731b50e31..465baf90c3 100644 --- a/src/test/setup_common.h +++ b/src/test/setup_common.h @@ -76,6 +76,12 @@ struct TestingSetup : public BasicTestingSetup { ~TestingSetup(); }; +/** Identical to TestingSetup, but chain set to regtest */ +struct RegTestingSetup : public TestingSetup { + RegTestingSetup() + : TestingSetup{CBaseChainParams::REGTEST} {} +}; + class CBlock; struct CMutableTransaction; class CScript; @@ -84,7 +90,7 @@ class CScript; // Testing fixture that pre-creates a // 100-block REGTEST-mode block chain // -struct TestChain100Setup : public TestingSetup { +struct TestChain100Setup : public RegTestingSetup { TestChain100Setup(); // Create a new block with just given transactions, coinbase paying to diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp index 7b00222ab7..c1a6df72d5 100644 --- a/src/test/timedata_tests.cpp +++ b/src/test/timedata_tests.cpp @@ -5,6 +5,7 @@ #include <netaddress.h> #include <noui.h> +#include <test/lib/logging.h> #include <test/setup_common.h> #include <timedata.h> #include <warnings.h> @@ -59,9 +60,10 @@ BOOST_AUTO_TEST_CASE(addtimedata) MultiAddTimeData(3, DEFAULT_MAX_TIME_ADJUSTMENT + 1); // Filter size is 1 + 3 = 4: It is always initialized with a single element (offset 0) - noui_suppress(); - MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5 - noui_reconnect(); + { + ASSERT_DEBUG_LOG("Please check that your computer's date and time are correct!"); + MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5 + } BOOST_CHECK(GetWarnings("gui").find("clock is wrong") != std::string::npos); diff --git a/src/test/util/str.cpp b/src/test/util/str.cpp new file mode 100644 index 0000000000..c517fe44d9 --- /dev/null +++ b/src/test/util/str.cpp @@ -0,0 +1,21 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/str.h> + +#include <cstdint> +#include <string> + +bool CaseInsensitiveEqual(const std::string& s1, const std::string& s2) +{ + if (s1.size() != s2.size()) return false; + for (size_t i = 0; i < s1.size(); ++i) { + char c1 = s1[i]; + if (c1 >= 'A' && c1 <= 'Z') c1 -= ('A' - 'a'); + char c2 = s2[i]; + if (c2 >= 'A' && c2 <= 'Z') c2 -= ('A' - 'a'); + if (c1 != c2) return false; + } + return true; +} diff --git a/src/test/util/str.h b/src/test/util/str.h new file mode 100644 index 0000000000..63629501e8 --- /dev/null +++ b/src/test/util/str.h @@ -0,0 +1,12 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_STR_H +#define BITCOIN_TEST_UTIL_STR_H + +#include <string> + +bool CaseInsensitiveEqual(const std::string& s1, const std::string& s2); + +#endif // BITCOIN_TEST_UTIL_STR_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 569ce53092..45f4e3a51b 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -231,6 +231,60 @@ BOOST_AUTO_TEST_CASE(util_ParseParameters) BOOST_CHECK(testArgs.GetArgs("-ccc").size() == 2); } +static void TestParse(const std::string& str, bool expected_bool, int64_t expected_int) +{ + TestArgsManager test; + test.SetupArgs({{"-value", ArgsManager::ALLOW_ANY}}); + std::string arg = "-value=" + str; + const char* argv[] = {"ignored", arg.c_str()}; + std::string error; + BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), expected_bool); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), expected_bool); + BOOST_CHECK_EQUAL(test.GetArg("-value", 99998), expected_int); + BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), expected_int); +} + +// Test bool and int parsing. +BOOST_AUTO_TEST_CASE(util_ArgParsing) +{ + // Some of these cases could be ambiguous or surprising to users, and might + // be worth triggering errors or warnings in the future. But for now basic + // test coverage is useful to avoid breaking backwards compatibility + // unintentionally. + TestParse("", true, 0); + TestParse(" ", false, 0); + TestParse("0", false, 0); + TestParse("0 ", false, 0); + TestParse(" 0", false, 0); + TestParse("+0", false, 0); + TestParse("-0", false, 0); + TestParse("5", true, 5); + TestParse("5 ", true, 5); + TestParse(" 5", true, 5); + TestParse("+5", true, 5); + TestParse("-5", true, -5); + TestParse("0 5", false, 0); + TestParse("5 0", true, 5); + TestParse("050", true, 50); + TestParse("0.", false, 0); + TestParse("5.", true, 5); + TestParse("0.0", false, 0); + TestParse("0.5", false, 0); + TestParse("5.0", true, 5); + TestParse("5.5", true, 5); + TestParse("x", false, 0); + TestParse("x0", false, 0); + TestParse("x5", false, 0); + TestParse("0x", false, 0); + TestParse("5x", true, 5); + TestParse("0x5", false, 0); + TestParse("false", false, 0); + TestParse("true", false, 0); + TestParse("yes", false, 0); + TestParse("no", false, 0); +} + BOOST_AUTO_TEST_CASE(util_GetBoolArg) { TestArgsManager testArgs; @@ -890,6 +944,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) desc += " "; desc += argstr + 1; conf += argstr + 1; + conf += "\n"; } std::istringstream conf_stream(conf); BOOST_CHECK(parser.ReadConfigStream(conf_stream, "filepath", error)); @@ -928,7 +983,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) // Results file is formatted like: // // <input> || <output> - BOOST_CHECK_EQUAL(out_sha_hex, "94b4ad55c8ac639a56b93e36f7e32e4c611fd7d7dd7b2be6a71707b1eadcaec7"); + BOOST_CHECK_EQUAL(out_sha_hex, "f0b3a3c29869edc765d579c928f7f1690a71fbb673b49ccf39cbc4de18156a0d"); } BOOST_AUTO_TEST_CASE(util_FormatMoney) diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index aca9f475ac..ae998e92a5 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -18,13 +18,9 @@ #include <thread> -struct RegtestingSetup : public TestingSetup { - RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {} -}; - static const std::vector<unsigned char> V_OP_TRUE{OP_TRUE}; -BOOST_FIXTURE_TEST_SUITE(validation_block_tests, RegtestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_block_tests, RegTestingSetup) struct TestSubscriber : public CValidationInterface { uint256 m_expected_tip; diff --git a/src/util/system.h b/src/util/system.h index 908a3c407d..7452f186e6 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -265,7 +265,7 @@ public: void ForceSetArg(const std::string& strArg, const std::string& strValue); /** - * Looks for -regtest, -testnet and returns the appropriate BIP70 chain name. + * Returns the appropriate chain name from the program arguments. * @return CBaseChainParams::MAIN by default; raises runtime error if an invalid combination is given. */ std::string GetChainName() const; diff --git a/src/validation.cpp b/src/validation.cpp index ca6d2176b3..01803223d1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -18,6 +18,8 @@ #include <flatfile.h> #include <hash.h> #include <index/txindex.h> +#include <logging.h> +#include <logging/timer.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -733,7 +735,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // To check these we first check if we meet the RBF criteria, above, and increment the descendant // limits by the direct conflict and its descendants (as these are recalculated in // CalculateMempoolAncestors by assuming the new transaction being added is a new descendant, with no - // removals, of each parent's existing dependant set). The ancestor count limits are unmodified (as + // removals, of each parent's existing dependent set). The ancestor count limits are unmodified (as // the ancestor limits should be the same for both our new transaction and any conflicts). // We don't bother incrementing m_limit_descendants by the full removal count as that limit never comes // into force here (as we're only adding a single transaction). @@ -2199,6 +2201,10 @@ bool CChainState::FlushStateToDisk( static int64_t nLastFlush = 0; std::set<int> setFilesToPrune; bool full_flush_completed = false; + + const size_t coins_count = CoinsTip().GetCacheSize(); + const size_t coins_mem_usage = CoinsTip().DynamicMemoryUsage(); + try { { bool fFlushForPrune = false; @@ -2206,8 +2212,12 @@ bool CChainState::FlushStateToDisk( LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { if (nManualPruneHeight > 0) { + LOG_TIME_MILLIS("find files to prune (manual)", BCLog::BENCH); + FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight); } else { + LOG_TIME_MILLIS("find files to prune", BCLog::BENCH); + FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight()); fCheckForPruning = false; } @@ -2246,10 +2256,17 @@ bool CChainState::FlushStateToDisk( if (!CheckDiskSpace(GetBlocksDir())) { return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); } - // First make sure all block and undo data is flushed to disk. - FlushBlockFile(); + { + LOG_TIME_MILLIS("write block and undo data to disk", BCLog::BENCH); + + // First make sure all block and undo data is flushed to disk. + FlushBlockFile(); + } + // Then update all block file information (which may refer to block and undo files). { + LOG_TIME_MILLIS("write block index to disk", BCLog::BENCH); + std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; vFiles.reserve(setDirtyFileInfo.size()); for (std::set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { @@ -2267,12 +2284,18 @@ bool CChainState::FlushStateToDisk( } } // Finally remove any pruned files - if (fFlushForPrune) + if (fFlushForPrune) { + LOG_TIME_MILLIS("unlink pruned files", BCLog::BENCH); + UnlinkPrunedFiles(setFilesToPrune); + } nLastWrite = nNow; } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush && !CoinsTip().GetBestBlock().IsNull()) { + LOG_TIME_SECONDS(strprintf("write coins cache to disk (%d coins, %.2fkB)", + coins_count, coins_mem_usage / 1000)); + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already diff --git a/src/validation.h b/src/validation.h index 7f9582adfd..e9127040bf 100644 --- a/src/validation.h +++ b/src/validation.h @@ -21,6 +21,7 @@ #include <txmempool.h> // For CTxMemPool::cs #include <txdb.h> #include <versionbits.h> +#include <serialize.h> #include <algorithm> #include <atomic> diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index f7353ebbbb..6995974f08 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -87,15 +87,6 @@ static void RescanWallet(CWallet& wallet, const WalletRescanReserver& reserver, } } -static LegacyScriptPubKeyMan& GetLegacyScriptPubKeyMan(CWallet& wallet) -{ - LegacyScriptPubKeyMan* spk_man = wallet.GetLegacyScriptPubKeyMan(); - if (!spk_man) { - throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); - } - return *spk_man; -} - UniValue importprivkey(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -134,7 +125,7 @@ UniValue importprivkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); } - LegacyScriptPubKeyMan& spk_man = GetLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet); WalletRescanReserver reserver(pwallet); bool fRescan = true; @@ -168,7 +159,7 @@ UniValue importprivkey(const JSONRPCRequest& request) if (!key.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); CPubKey pubkey = key.GetPubKey(); - assert(key.VerifyPubKey(pubkey)); + CHECK_NONFATAL(key.VerifyPubKey(pubkey)); CKeyID vchAddress = pubkey.GetID(); { pwallet->MarkDirty(); @@ -262,7 +253,7 @@ UniValue importaddress(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan& spk_man = GetLegacyScriptPubKeyMan(*pwallet); + EnsureLegacyScriptPubKeyMan(*pwallet); std::string strLabel; if (!request.params[1].isNull()) @@ -465,7 +456,7 @@ UniValue importpubkey(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan& spk_man = GetLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet); std::string strLabel; if (!request.params[1].isNull()) @@ -549,7 +540,7 @@ UniValue importwallet(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan& spk_man = GetLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet); if (pwallet->chain().havePruned()) { // Exit early and print an error. @@ -639,7 +630,7 @@ UniValue importwallet(const JSONRPCRequest& request) std::string label = std::get<3>(key_tuple); CPubKey pubkey = key.GetPubKey(); - assert(key.VerifyPubKey(pubkey)); + CHECK_NONFATAL(key.VerifyPubKey(pubkey)); CKeyID keyid = pubkey.GetID(); pwallet->WalletLogPrintf("Importing %s...\n", EncodeDestination(PKHash(keyid))); @@ -708,7 +699,7 @@ UniValue dumpprivkey(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan& spk_man = GetLegacyScriptPubKeyMan(*wallet); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -759,7 +750,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan& spk_man = GetLegacyScriptPubKeyMan(*wallet); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -906,7 +897,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d case TX_SCRIPTHASH: { if (script_ctx == ScriptContext::P2SH) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2SH inside another P2SH"); if (script_ctx == ScriptContext::WITNESS_V0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2SH inside a P2WSH"); - assert(script_ctx == ScriptContext::TOP); + CHECK_NONFATAL(script_ctx == ScriptContext::TOP); CScriptID id = CScriptID(uint160(solverdata[0])); auto subscript = std::move(import_data.redeemscript); // Remove redeemscript from import_data to check for superfluous script later. if (!subscript) return "missing redeemscript"; @@ -1346,7 +1337,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) RPCTypeCheck(mainRequest.params, {UniValue::VARR, UniValue::VOBJ}); - LegacyScriptPubKeyMan& spk_man = GetLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet); const UniValue& requests = mainRequest.params[0]; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index bfa4cf2bbe..6098a08292 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -124,6 +124,15 @@ void EnsureWalletIsUnlocked(const CWallet* pwallet) } } +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet) +{ + LegacyScriptPubKeyMan* spk_man = wallet.GetLegacyScriptPubKeyMan(); + if (!spk_man) { + throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); + } + return *spk_man; +} + static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx, UniValue& entry) { int confirms = wtx.GetDepthInMainChain(locked_chain); @@ -136,7 +145,7 @@ static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& lo entry.pushKV("blockindex", wtx.m_confirm.nIndex); int64_t block_time; bool found_block = chain.findBlock(wtx.m_confirm.hashBlock, nullptr /* block */, &block_time); - assert(found_block); + CHECK_NONFATAL(found_block); entry.pushKV("blocktime", block_time); } else { entry.pushKV("trusted", wtx.IsTrusted(locked_chain)); @@ -966,10 +975,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan* spk_man = pwallet->GetLegacyScriptPubKeyMan(); - if (!spk_man) { - throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); - } + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -987,7 +993,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) if (IsHex(keys_or_addrs[i].get_str()) && (keys_or_addrs[i].get_str().length() == 66 || keys_or_addrs[i].get_str().length() == 130)) { pubkeys.push_back(HexToPubKey(keys_or_addrs[i].get_str())); } else { - pubkeys.push_back(AddrToPubKey(spk_man, keys_or_addrs[i].get_str())); + pubkeys.push_back(AddrToPubKey(&spk_man, keys_or_addrs[i].get_str())); } } @@ -1000,7 +1006,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) // Construct using pay-to-script-hash: CScript inner; - CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, *spk_man, inner); + CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, spk_man, inner); pwallet->SetAddressBook(dest, label, "send"); UniValue result(UniValue::VOBJ); @@ -1598,7 +1604,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { CWalletTx tx = pairWtx.second; - if (depth == -1 || tx.GetDepthInMainChain(*locked_chain) < depth) { + if (depth == -1 || abs(tx.GetDepthInMainChain(*locked_chain)) < depth) { ListTransactions(*locked_chain, pwallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); } } @@ -2943,7 +2949,7 @@ static UniValue listunspent(const JSONRPCRequest& request) CTxDestination witness_destination; if (redeemScript.IsPayToWitnessScriptHash()) { bool extracted = ExtractDestination(redeemScript, witness_destination); - assert(extracted); + CHECK_NONFATAL(extracted); // Also return the witness script const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(witness_destination); CScriptID id; @@ -3756,26 +3762,16 @@ UniValue getaddressinfo(const JSONRPCRequest& request) ret.pushKV("label", pwallet->mapAddressBook[dest].name); } ret.pushKV("ischange", pwallet->IsChange(scriptPubKey)); - const CKeyMetadata* meta = nullptr; - CKeyID key_id = GetKeyForDestination(*provider, dest); - if (!key_id.IsNull()) { - auto it = pwallet->mapKeyMetadata.find(key_id); - if (it != pwallet->mapKeyMetadata.end()) { - meta = &it->second; - } - } - if (!meta) { - auto it = pwallet->m_script_metadata.find(CScriptID(scriptPubKey)); - if (it != pwallet->m_script_metadata.end()) { - meta = &it->second; - } - } - if (meta) { - ret.pushKV("timestamp", meta->nCreateTime); - if (meta->has_key_origin) { - ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); - ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); - ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint, meta->key_origin.fingerprint + 4)); + + ScriptPubKeyMan* spk_man = pwallet->GetScriptPubKeyMan(); + if (spk_man) { + if (const CKeyMetadata* meta = spk_man->GetMetadata(dest)) { + ret.pushKV("timestamp", meta->nCreateTime); + if (meta->has_key_origin) { + ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); + ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); + ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint, meta->key_origin.fingerprint + 4)); + } } } @@ -3833,7 +3829,7 @@ static UniValue getaddressesbylabel(const JSONRPCRequest& request) // address strings, but build a separate set as a precaution just in // case it does. bool unique = addresses.emplace(address).second; - assert(unique); + CHECK_NONFATAL(unique); // UniValue::pushKV checks if the key exists in O(N) // and since duplicate addresses are unexpected (checked with // std::set in O(log(N))), UniValue::__pushKV is used instead, @@ -3935,10 +3931,7 @@ UniValue sethdseed(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan* spk_man = pwallet->GetLegacyScriptPubKeyMan(); - if (!spk_man) { - throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); - } + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); if (pwallet->chain().isInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Cannot set a new HD seed while still in Initial Block Download"); @@ -3965,22 +3958,22 @@ UniValue sethdseed(const JSONRPCRequest& request) CPubKey master_pub_key; if (request.params[1].isNull()) { - master_pub_key = spk_man->GenerateNewSeed(); + master_pub_key = spk_man.GenerateNewSeed(); } else { CKey key = DecodeSecret(request.params[1].get_str()); if (!key.IsValid()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key"); } - if (HaveKey(*spk_man, key)) { + if (HaveKey(spk_man, key)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Already have this key (either as an HD seed or as a loose private key)"); } - master_pub_key = spk_man->DeriveNewSeed(key); + master_pub_key = spk_man.DeriveNewSeed(key); } - spk_man->SetHDSeed(master_pub_key); - if (flush_key_pool) spk_man->NewKeyPool(); + spk_man.SetHDSeed(master_pub_key); + if (flush_key_pool) spk_man.NewKeyPool(); return NullUniValue; } diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 31d3f7a5f9..becca455f6 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -12,6 +12,7 @@ class CRPCTable; class CWallet; class JSONRPCRequest; +class LegacyScriptPubKeyMan; class UniValue; struct PartiallySignedTransaction; class CTransaction; @@ -40,6 +41,7 @@ std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& reques std::string HelpRequiringPassphrase(const CWallet*); void EnsureWalletIsUnlocked(const CWallet*); bool EnsureWalletIsAvailable(const CWallet*, bool avoidException); +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet); UniValue getaddressinfo(const JSONRPCRequest& request); UniValue signrawtransactionwithwallet(const JSONRPCRequest& request); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 259bfcd76d..3eaaf3786c 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -11,11 +11,10 @@ #include <wallet/scriptpubkeyman.h> #include <wallet/wallet.h> -bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error) +bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { - LOCK(cs_wallet); error.clear(); - TopUpKeyPool(); + TopUp(); // Generate a new key that is added to wallet CPubKey new_key; @@ -25,8 +24,6 @@ bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, const std:: } LearnRelatedScripts(new_key, type); dest = GetDestinationForKey(new_key, type); - - m_wallet.SetAddressBook(dest, label, "receive"); return true; } @@ -265,6 +262,41 @@ bool LegacyScriptPubKeyMan::EncryptKeys(CKeyingMaterial& vMasterKeyIn) return true; } +bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, int64_t& index, CKeyPool& keypool) +{ + if (!ReserveKeyFromKeyPool(index, keypool, internal)) { + return false; + } + return true; +} + +void LegacyScriptPubKeyMan::KeepDestination(int64_t index) +{ + KeepKey(index); +} + +void LegacyScriptPubKeyMan::ReturnDestination(int64_t index, bool internal, const CPubKey& pubkey) +{ + ReturnKey(index, internal, pubkey); +} + +void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) +{ + AssertLockHeld(cs_wallet); + // extract addresses and check if they match with an unused keypool key + for (const auto& keyid : GetAffectedKeys(script, *this)) { + std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); + if (mi != m_pool_key_to_index.end()) { + WalletLogPrintf("%s: Detected a used keypool key, mark all keypool key up to this key as used\n", __func__); + MarkReserveKeysAsUsed(mi->second); + + if (!TopUp()) { + WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); + } + } + } +} + void LegacyScriptPubKeyMan::UpgradeKeyMetadata() { AssertLockHeld(cs_wallet); @@ -298,8 +330,19 @@ void LegacyScriptPubKeyMan::UpgradeKeyMetadata() } } } - batch.reset(); //write before setting the flag - m_storage.SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); +} + +bool LegacyScriptPubKeyMan::SetupGeneration(bool force) +{ + if ((CanGenerateKeys() && !force) || m_storage.IsLocked()) { + return false; + } + + SetHDSeed(GenerateNewSeed()); + if (!NewKeyPool()) { + return false; + } + return true; } bool LegacyScriptPubKeyMan::IsHDEnabled() const @@ -324,6 +367,58 @@ bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) return keypool_has_keys; } +bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) +{ + AssertLockHeld(cs_wallet); + error = ""; + bool hd_upgrade = false; + bool split_upgrade = false; + if (m_storage.CanSupportFeature(FEATURE_HD) && !IsHDEnabled()) { + WalletLogPrintf("Upgrading wallet to HD\n"); + m_storage.SetMinVersion(FEATURE_HD); + + // generate a new master key + CPubKey masterPubKey = GenerateNewSeed(); + SetHDSeed(masterPubKey); + hd_upgrade = true; + } + // Upgrade to HD chain split if necessary + if (m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { + WalletLogPrintf("Upgrading wallet to use HD chain split\n"); + m_storage.SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); + split_upgrade = FEATURE_HD_SPLIT > prev_version; + } + // Mark all keys currently in the keypool as pre-split + if (split_upgrade) { + MarkPreSplitKeys(); + } + // Regenerate the keypool if upgraded to HD + if (hd_upgrade) { + if (!TopUp()) { + error = _("Unable to generate keys").translated; + return false; + } + } + return true; +} + +bool LegacyScriptPubKeyMan::HavePrivateKeys() const +{ + LOCK(cs_KeyStore); + return !mapKeys.empty() || !mapCryptedKeys.empty(); +} + +void LegacyScriptPubKeyMan::RewriteDB() +{ + AssertLockHeld(cs_wallet); + setInternalKeyPool.clear(); + setExternalKeyPool.clear(); + m_pool_key_to_index.clear(); + // Note: can't top-up keypool here, because wallet is locked. + // User will be prompted to unlock wallet the next operation + // that requires a new key. +} + static int64_t GetOldestKeyTimeInPool(const std::set<int64_t>& setKeyPool, WalletBatch& batch) { if (setKeyPool.empty()) { return GetTime(); @@ -362,6 +457,39 @@ size_t LegacyScriptPubKeyMan::KeypoolCountExternalKeys() return setExternalKeyPool.size() + set_pre_split_keypool.size(); } +unsigned int LegacyScriptPubKeyMan::GetKeyPoolSize() const +{ + AssertLockHeld(cs_wallet); + return setInternalKeyPool.size() + setExternalKeyPool.size(); +} + +int64_t LegacyScriptPubKeyMan::GetTimeFirstKey() const +{ + AssertLockHeld(cs_wallet); + return nTimeFirstKey; +} + +const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const +{ + AssertLockHeld(cs_wallet); + + CKeyID key_id = GetKeyForDestination(*this, dest); + if (!key_id.IsNull()) { + auto it = mapKeyMetadata.find(key_id); + if (it != mapKeyMetadata.end()) { + return &it->second; + } + } + + CScript scriptPubKey = GetScriptForDestination(dest); + auto it = m_script_metadata.find(CScriptID(scriptPubKey)); + if (it != m_script_metadata.end()) { + return &it->second; + } + + return nullptr; +} + /** * Update wallet first key creation time. This should be called whenever keys * are added to the wallet, with the oldest key creation time. @@ -378,6 +506,11 @@ void LegacyScriptPubKeyMan::UpdateTimeFirstKey(int64_t nCreateTime) } } +bool LegacyScriptPubKeyMan::LoadKey(const CKey& key, const CPubKey &pubkey) +{ + return AddKeyPubKeyInner(key, pubkey); +} + bool LegacyScriptPubKeyMan::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) { WalletBatch batch(m_storage.GetDatabase()); @@ -420,7 +553,7 @@ bool LegacyScriptPubKeyMan::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& s secret.GetPrivKey(), mapKeyMetadata[pubkey.GetID()]); } - m_storage.UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); + m_storage.UnsetBlankWalletFlag(batch); return true; } @@ -577,7 +710,7 @@ bool LegacyScriptPubKeyMan::AddWatchOnlyWithDB(WalletBatch &batch, const CScript UpdateTimeFirstKey(meta.nCreateTime); NotifyWatchonlyChanged(true); if (batch.WriteWatchOnly(dest, meta)) { - m_storage.UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); + m_storage.UnsetBlankWalletFlag(batch); return true; } return false; @@ -855,7 +988,8 @@ void LegacyScriptPubKeyMan::SetHDSeed(const CPubKey& seed) newHdChain.seed_id = seed.GetID(); SetHDChain(newHdChain, false); NotifyCanGetAddressesChanged(); - m_wallet.UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); + WalletBatch batch(m_storage.GetDatabase()); + m_storage.UnsetBlankWalletFlag(batch); } /** @@ -888,7 +1022,7 @@ bool LegacyScriptPubKeyMan::NewKeyPool() m_pool_key_to_index.clear(); - if (!TopUpKeyPool()) { + if (!TopUp()) { return false; } WalletLogPrintf("LegacyScriptPubKeyMan::NewKeyPool rewrote keypool\n"); @@ -896,7 +1030,7 @@ bool LegacyScriptPubKeyMan::NewKeyPool() return true; } -bool LegacyScriptPubKeyMan::TopUpKeyPool(unsigned int kpSize) +bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) { if (!CanGenerateKeys()) { return false; @@ -1013,7 +1147,7 @@ bool LegacyScriptPubKeyMan::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& key { LOCK(cs_wallet); - TopUpKeyPool(); + TopUp(); bool fReturningInternal = fRequestedInternal; fReturningInternal &= (IsHDEnabled() && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) || m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); @@ -1134,7 +1268,7 @@ bool LegacyScriptPubKeyMan::AddCScriptWithDB(WalletBatch& batch, const CScript& if (!FillableSigningProvider::AddCScript(redeemScript)) return false; if (batch.WriteCScript(Hash160(redeemScript), redeemScript)) { - m_storage.UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); + m_storage.UnsetBlankWalletFlag(batch); return true; } return false; @@ -1229,7 +1363,7 @@ bool LegacyScriptPubKeyMan::ImportPubKeys(const std::vector<CKeyID>& ordered_pub return true; } -bool LegacyScriptPubKeyMan::ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool apply_label, const int64_t timestamp) +bool LegacyScriptPubKeyMan::ImportScriptPubKeys(const std::set<CScript>& script_pub_keys, const bool have_solving_data, const int64_t timestamp) { WalletBatch batch(m_storage.GetDatabase()); for (const CScript& script : script_pub_keys) { @@ -1238,11 +1372,6 @@ bool LegacyScriptPubKeyMan::ImportScriptPubKeys(const std::string& label, const return false; } } - CTxDestination dest; - ExtractDestination(script, dest); - if (apply_label && IsValidDestination(dest)) { - m_wallet.SetAddressBookWithDB(batch, dest, label, "receive"); - } } return true; } diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 16a5c9b979..4f17156792 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -28,8 +28,7 @@ public: virtual const std::string GetDisplayName() const = 0; virtual WalletDatabase& GetDatabase() = 0; virtual bool IsWalletFlagSet(uint64_t) const = 0; - virtual void SetWalletFlag(uint64_t) = 0; - virtual void UnsetWalletFlagWithDB(WalletBatch&, uint64_t) = 0; + virtual void UnsetBlankWalletFlag(WalletBatch&) = 0; virtual bool CanSupportFeature(enum WalletFeature) const = 0; virtual void SetMinVersion(enum WalletFeature, WalletBatch* = nullptr, bool = false) = 0; virtual bool IsLocked() const = 0; @@ -38,6 +37,8 @@ public: //! Default for -keypool static const unsigned int DEFAULT_KEYPOOL_SIZE = 1000; +std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& provider); + /** A key from a CWallet's keypool * * The wallet holds one (for pre HD-split wallets) or several keypools. These @@ -145,41 +146,69 @@ protected: public: ScriptPubKeyMan(WalletStorage& storage) : m_storage(storage) {} + virtual ~ScriptPubKeyMan() {}; + virtual bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { return false; } + virtual isminetype IsMine(const CScript& script) const { return ISMINE_NO; } + + virtual bool GetReservedDestination(const OutputType type, bool internal, int64_t& index, CKeyPool& keypool) { return false; } + virtual void KeepDestination(int64_t index) {} + virtual void ReturnDestination(int64_t index, bool internal, const CPubKey& pubkey) {} + + virtual bool TopUp(unsigned int size = 0) { return false; } + + //! Mark unused addresses as being used + virtual void MarkUnusedAddresses(const CScript& script) {} + + /** Sets up the key generation stuff, i.e. generates new HD seeds and sets them as active. + * Returns false if already setup or setup fails, true if setup is successful + * Set force=true to make it re-setup if already setup, used for upgrades + */ + virtual bool SetupGeneration(bool force = false) { return false; } + + /* Returns true if HD is enabled */ + virtual bool IsHDEnabled() const { return false; } + + /* Returns true if the wallet can give out new addresses. This means it has keys in the keypool or can generate new keys */ + virtual bool CanGetAddresses(bool internal = false) { return false; } + + /** Upgrades the wallet to the specified version */ + virtual bool Upgrade(int prev_version, std::string& error) { return false; } + + virtual bool HavePrivateKeys() const { return false; } + + //! The action to do when the DB needs rewrite + virtual void RewriteDB() {} + + virtual int64_t GetOldestKeyPoolTime() { return GetTime(); } + + virtual size_t KeypoolCountExternalKeys() { return 0; } + virtual unsigned int GetKeyPoolSize() const { return 0; } + + virtual int64_t GetTimeFirstKey() const { return 0; } + + //! Return address metadata + virtual const CKeyMetadata* GetMetadata(const CTxDestination& dest) const { return nullptr; } }; class LegacyScriptPubKeyMan : public ScriptPubKeyMan, public FillableSigningProvider { private: - using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; using WatchOnlySet = std::set<CScript>; using WatchKeyMap = std::map<CKeyID, CPubKey>; - //! will encrypt previously unencrypted keys - bool EncryptKeys(CKeyingMaterial& vMasterKeyIn); + WalletBatch *encrypted_batch GUARDED_BY(cs_wallet) = nullptr; + + using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; CryptedKeyMap mapCryptedKeys GUARDED_BY(cs_KeyStore); WatchOnlySet setWatchOnly GUARDED_BY(cs_KeyStore); WatchKeyMap mapWatchKeys GUARDED_BY(cs_KeyStore); - bool AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); - bool AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey); - - WalletBatch *encrypted_batch GUARDED_BY(cs_wallet) = nullptr; - - /* the HD chain data model (external chain counters) */ - CHDChain hdChain; - - /* HD derive new child key (on internal or external chain) */ - void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); - int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; - std::map<CKeyID, int64_t> m_pool_key_to_index; - int64_t nTimeFirstKey GUARDED_BY(cs_wallet) = 0; + bool AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey); + bool AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); + /** * Private version of AddWatchOnly method which does not accept a * timestamp, and which will reset the wallet's nTimeFirstKey value to 1 if @@ -192,26 +221,91 @@ private: bool AddWatchOnly(const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool AddWatchOnlyInMem(const CScript &dest); - - /** Add a KeyOriginInfo to the wallet */ - bool AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info); + //! Adds a watch-only address to the store, and saves it to disk. + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a key to the store, and saves it to disk. bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! Adds a watch-only address to the store, and saves it to disk. - bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); //! Adds a script to the store and saves it to disk bool AddCScriptWithDB(WalletBatch& batch, const CScript& script); - public: + /** Add a KeyOriginInfo to the wallet */ + bool AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info); + + /* the HD chain data model (external chain counters) */ + CHDChain hdChain; + + /* HD derive new child key (on internal or external chain) */ + void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_wallet); + std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_wallet); + std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); + int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; + std::map<CKeyID, int64_t> m_pool_key_to_index; + //! Fetches a key from the keypool bool GetKeyFromPool(CPubKey &key, bool internal = false); - void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + /** + * Reserves a key from the keypool and sets nIndex to its index + * + * @param[out] nIndex the index of the key in keypool + * @param[out] keypool the keypool the key was drawn from, which could be the + * the pre-split pool if present, or the internal or external pool + * @param fRequestedInternal true if the caller would like the key drawn + * from the internal keypool, false if external is preferred + * + * @return true if succeeded, false if failed due to empty keypool + * @throws std::runtime_error if keypool read failed, key was invalid, + * was not found in the wallet, or was misclassified in the internal + * or external keypool + */ + bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); + + void KeepKey(int64_t nIndex); + void ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey); + +public: + bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; + isminetype IsMine(const CScript& script) const override; + + //! will encrypt previously unencrypted keys + bool EncryptKeys(CKeyingMaterial& vMasterKeyIn); + + bool GetReservedDestination(const OutputType type, bool internal, int64_t& index, CKeyPool& keypool) override; + void KeepDestination(int64_t index) override; + void ReturnDestination(int64_t index, bool internal, const CPubKey& pubkey) override; + + bool TopUp(unsigned int size = 0) override; + + void MarkUnusedAddresses(const CScript& script) override; + + //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo + void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + bool IsHDEnabled() const override; + + bool SetupGeneration(bool force = false) override; + + bool Upgrade(int prev_version, std::string& error) override; + + bool HavePrivateKeys() const override; + + void RewriteDB() override; + + int64_t GetOldestKeyPoolTime() override; + size_t KeypoolCountExternalKeys() override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + unsigned int GetKeyPoolSize() const override; + + int64_t GetTimeFirstKey() const override; + + const CKeyMetadata* GetMetadata(const CTxDestination& dest) const override; + + bool CanGetAddresses(bool internal = false) override; // Map from Key ID to key metadata. std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_wallet); @@ -219,94 +313,60 @@ private: // Map from Script ID to key metadata (for watch-only keys). std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_wallet); - /** - * keystore implementation - * Generate a new key - */ - CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a key to the store, and saves it to disk. bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a key to the store, without saving it to disk (used by LoadWallet) - bool LoadKey(const CKey& key, const CPubKey &pubkey) { return AddKeyPubKeyInner(key, pubkey); } - //! Load metadata (used by LoadWallet) - void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo - void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - + bool LoadKey(const CKey& key, const CPubKey &pubkey); //! Adds an encrypted key to the store, and saves it to disk. bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); //! Adds an encrypted key to the store, without saving it to disk (used by LoadWallet) bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); - bool GetKey(const CKeyID &address, CKey& keyOut) const override; - bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; - bool HaveKey(const CKeyID &address) const override; - std::set<CKeyID> GetKeys() const override; - bool AddCScript(const CScript& redeemScript) override; + void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Adds a CScript to the store bool LoadCScript(const CScript& redeemScript); + //! Load metadata (used by LoadWallet) + void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Generate a new key + CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + /* Set the HD chain model (chain child index counters) */ + void SetHDChain(const CHDChain& chain, bool memonly); + const CHDChain& GetHDChain() const { return hdChain; } - //! Adds a watch-only address to the store, and saves it to disk. - bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool RemoveWatchOnly(const CScript &dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a watch-only address to the store, without saving it to disk (used by LoadWallet) bool LoadWatchOnly(const CScript &dest); //! Returns whether the watch-only script is in the wallet bool HaveWatchOnly(const CScript &dest) const; //! Returns whether there are any watch-only things in the wallet bool HaveWatchOnly() const; + //! Remove a watch only script from the keystore + bool RemoveWatchOnly(const CScript &dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Fetches a pubkey from mapWatchKeys if it exists there bool GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const; - bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool apply_label, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /* SigningProvider overrides */ + bool HaveKey(const CKeyID &address) const override; + bool GetKey(const CKeyID &address, CKey& keyOut) const override; + bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; + bool AddCScript(const CScript& redeemScript) override; + bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; + //! Load a keypool entry + void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool NewKeyPool(); - size_t KeypoolCountExternalKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool TopUpKeyPool(unsigned int kpSize = 0); - - /** - * Reserves a key from the keypool and sets nIndex to its index - * - * @param[out] nIndex the index of the key in keypool - * @param[out] keypool the keypool the key was drawn from, which could be the - * the pre-split pool if present, or the internal or external pool - * @param fRequestedInternal true if the caller would like the key drawn - * from the internal keypool, false if external is preferred - * - * @return true if succeeded, false if failed due to empty keypool - * @throws std::runtime_error if keypool read failed, key was invalid, - * was not found in the wallet, or was misclassified in the internal - * or external keypool - */ - bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); - void KeepKey(int64_t nIndex); - void ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey); - int64_t GetOldestKeyPoolTime(); - /** - * Marks all keys in the keypool up to and including reserve_key as used. - */ - void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - const std::map<CKeyID, int64_t>& GetAllReserveKeys() const { return m_pool_key_to_index; } - bool GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error); - - isminetype IsMine(const CScript& script) const; - - /* Set the HD chain model (chain child index counters) */ - void SetHDChain(const CHDChain& chain, bool memonly); - const CHDChain& GetHDChain() const { return hdChain; } + void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /* Returns true if HD is enabled */ - bool IsHDEnabled() const; + bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportScriptPubKeys(const std::set<CScript>& script_pub_keys, const bool have_solving_data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /* Returns true if the wallet can generate new keys */ bool CanGenerateKeys(); - /* Returns true if the wallet can give out new addresses. This means it has keys in the keypool or can generate new keys */ - bool CanGetAddresses(bool internal = false); - /* Generates a new HD seed (will not be activated) */ CPubKey GenerateNewSeed(); @@ -333,9 +393,13 @@ private: */ void LearnAllRelatedScripts(const CPubKey& key); - /** Implement lookup of key origin information through wallet key metadata. */ - bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; + /** + * Marks all keys in the keypool up to and including reserve_key as used. + */ + void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + const std::map<CKeyID, int64_t>& GetAllReserveKeys() const { return m_pool_key_to_index; } + std::set<CKeyID> GetKeys() const override; // Temporary CWallet accessors and aliases. friend class CWallet; friend class ReserveDestination; diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 279542ffad..7a6907bc93 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -5,6 +5,7 @@ #include <boost/test/unit_test.hpp> #include <noui.h> +#include <test/lib/logging.h> #include <test/setup_common.h> #include <util/system.h> #include <wallet/test/init_test_fixture.h> @@ -34,28 +35,31 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_does_not_exist) { SetWalletDir(m_walletdir_path_cases["nonexistent"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("does not exist"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_directory) { SetWalletDir(m_walletdir_path_cases["file"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("is not a directory"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_relative) { SetWalletDir(m_walletdir_path_cases["relative"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("is a relative path"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 069ae57878..cdcb65e3c0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -210,9 +210,14 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& } // Set a seed for the wallet - CPubKey master_pub_key = wallet->m_spk_man->GenerateNewSeed(); - wallet->m_spk_man->SetHDSeed(master_pub_key); - wallet->m_spk_man->NewKeyPool(); + { + if (auto spk_man = wallet->m_spk_man.get()) { + if (!spk_man->SetupGeneration()) { + error = "Unable to generate initial keys"; + return WalletCreationStatus::CREATION_FAILED; + } + } + } // Relock the wallet wallet->Lock(); @@ -236,8 +241,6 @@ std::string COutput::ToString() const return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); } -std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& provider); - const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const { LOCK(cs_wallet); @@ -249,10 +252,15 @@ const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const void CWallet::UpgradeKeyMetadata() { + if (IsLocked() || IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { + return; + } + if (m_spk_man) { AssertLockHeld(m_spk_man->cs_wallet); m_spk_man->UpgradeKeyMetadata(); } + SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); } bool CWallet::Unlock(const SecureString& strWalletPassphrase, bool accept_no_keys) @@ -562,11 +570,13 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) Unlock(strWalletPassphrase); // if we are using HD, replace the HD seed with a new one - if (m_spk_man->IsHDEnabled()) { - m_spk_man->SetHDSeed(m_spk_man->GenerateNewSeed()); + if (auto spk_man = m_spk_man.get()) { + if (spk_man->IsHDEnabled()) { + if (!spk_man->SetupGeneration(true)) { + return false; + } + } } - - m_spk_man->NewKeyPool(); Lock(); // Need to completely rewrite the wallet file; if we don't, bdb might keep @@ -871,17 +881,8 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::St // loop though all outputs for (const CTxOut& txout: tx.vout) { - // extract addresses and check if they match with an unused keypool key - for (const auto& keyid : GetAffectedKeys(txout.scriptPubKey, *m_spk_man)) { - std::map<CKeyID, int64_t>::const_iterator mi = m_spk_man->m_pool_key_to_index.find(keyid); - if (mi != m_spk_man->m_pool_key_to_index.end()) { - WalletLogPrintf("%s: Detected a used keypool key, mark all keypool key up to this key as used\n", __func__); - MarkReserveKeysAsUsed(mi->second); - - if (!m_spk_man->TopUpKeyPool()) { - WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); - } - } + if (auto spk_man = m_spk_man.get()) { + spk_man->MarkUnusedAddresses(txout.scriptPubKey); } } @@ -1304,6 +1305,11 @@ void CWallet::UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag) throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } +void CWallet::UnsetBlankWalletFlag(WalletBatch& batch) +{ + UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); +} + bool CWallet::IsWalletFlagSet(uint64_t flag) const { return (m_wallet_flags & flag); @@ -1400,9 +1406,19 @@ bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScri return false; } AssertLockHeld(spk_man->cs_wallet); - if (!spk_man->ImportScriptPubKeys(label, script_pub_keys, have_solving_data, apply_label, timestamp)) { + if (!spk_man->ImportScriptPubKeys(script_pub_keys, have_solving_data, timestamp)) { return false; } + if (apply_label) { + WalletBatch batch(*database); + for (const CScript& script : script_pub_keys) { + CTxDestination dest; + ExtractDestination(script, dest); + if (IsValidDestination(dest)) { + SetAddressBookWithDB(batch, dest, label, "receive"); + } + } + } return true; } @@ -1836,32 +1852,37 @@ bool CWalletTx::InMempool() const bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain) const { + std::set<uint256> s; + return IsTrusted(locked_chain, s); +} + +bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const +{ // Quick answer in most cases - if (!locked_chain.checkFinalTx(*tx)) { - return false; - } + if (!locked_chain.checkFinalTx(*tx)) return false; int nDepth = GetDepthInMainChain(locked_chain); - if (nDepth >= 1) - return true; - if (nDepth < 0) - return false; - if (!pwallet->m_spend_zero_conf_change || !IsFromMe(ISMINE_ALL)) // using wtx's cached debit - return false; + if (nDepth >= 1) return true; + if (nDepth < 0) return false; + // using wtx's cached debit + if (!pwallet->m_spend_zero_conf_change || !IsFromMe(ISMINE_ALL)) return false; // Don't trust unconfirmed transactions from us unless they are in the mempool. - if (!InMempool()) - return false; + if (!InMempool()) return false; // Trusted if all inputs are from us and are in the mempool: for (const CTxIn& txin : tx->vin) { // Transactions not sent by us: not trusted const CWalletTx* parent = pwallet->GetWalletTx(txin.prevout.hash); - if (parent == nullptr) - return false; + if (parent == nullptr) return false; const CTxOut& parentOut = parent->tx->vout[txin.prevout.n]; - if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) - return false; + // Check that this specific input being spent is trusted + if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) return false; + // If we've already trusted this parent, continue + if (trusted_parents.count(parent->GetHash())) continue; + // Recurse to check that the parent is also trusted + if (!parent->IsTrusted(locked_chain, trusted_parents)) return false; + trusted_parents.insert(parent->GetHash()); } return true; } @@ -1947,10 +1968,11 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons { auto locked_chain = chain().lock(); LOCK(cs_wallet); + std::set<uint256> trusted_parents; for (const auto& entry : mapWallet) { const CWalletTx& wtx = entry.second; - const bool is_trusted{wtx.IsTrusted(*locked_chain)}; + const bool is_trusted{wtx.IsTrusted(*locked_chain, trusted_parents)}; const int tx_depth{wtx.GetDepthInMainChain(*locked_chain)}; const CAmount tx_credit_mine{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; @@ -1997,6 +2019,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< const int min_depth = {coinControl ? coinControl->m_min_depth : DEFAULT_MIN_DEPTH}; const int max_depth = {coinControl ? coinControl->m_max_depth : DEFAULT_MAX_DEPTH}; + std::set<uint256> trusted_parents; for (const auto& entry : mapWallet) { const uint256& wtxid = entry.first; @@ -2018,7 +2041,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = wtx.IsTrusted(locked_chain); + bool safeTx = wtx.IsTrusted(locked_chain, trusted_parents); // We should not consider coins from transactions that are replacing // other transactions. @@ -2889,12 +2912,9 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { if (database->Rewrite("\x04pool")) { - setInternalKeyPool.clear(); - setExternalKeyPool.clear(); - m_spk_man->m_pool_key_to_index.clear(); - // Note: can't top-up keypool here, because wallet is locked. - // User will be prompted to unlock wallet the next operation - // that requires a new key. + if (auto spk_man = m_spk_man.get()) { + spk_man->RewriteDB(); + } } } @@ -2926,12 +2946,9 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 { if (database->Rewrite("\x04pool")) { - setInternalKeyPool.clear(); - setExternalKeyPool.clear(); - m_spk_man->m_pool_key_to_index.clear(); - // Note: can't top-up keypool here, because wallet is locked. - // User will be prompted to unlock wallet the next operation - // that requires a new key. + if (auto spk_man = m_spk_man.get()) { + spk_man->RewriteDB(); + } } } @@ -2950,13 +2967,9 @@ DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) { if (database->Rewrite("\x04pool")) { - LOCK(cs_wallet); - setInternalKeyPool.clear(); - setExternalKeyPool.clear(); - m_spk_man->m_pool_key_to_index.clear(); - // Note: can't top-up keypool here, because wallet is locked. - // User will be prompted to unlock wallet the next operation - // that requires a new key. + if (auto spk_man = m_spk_man.get()) { + spk_man->RewriteDB(); + } } } @@ -3023,23 +3036,39 @@ size_t CWallet::KeypoolCountExternalKeys() return count; } +unsigned int CWallet::GetKeyPoolSize() const +{ + AssertLockHeld(cs_wallet); + + unsigned int count = 0; + if (auto spk_man = m_spk_man.get()) { + count += spk_man->GetKeyPoolSize(); + } + return count; +} + bool CWallet::TopUpKeyPool(unsigned int kpSize) { bool res = true; if (auto spk_man = m_spk_man.get()) { - res &= spk_man->TopUpKeyPool(kpSize); + res &= spk_man->TopUp(kpSize); } return res; } bool CWallet::GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error) { + LOCK(cs_wallet); error.clear(); bool result = false; auto spk_man = m_spk_man.get(); if (spk_man) { - result = spk_man->GetNewDestination(type, label, dest, error); + result = spk_man->GetNewDestination(type, dest, error); + } + if (result) { + SetAddressBook(dest, label, "receive"); } + return result; } @@ -3047,7 +3076,7 @@ bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& des { error.clear(); - m_spk_man->TopUpKeyPool(); + m_spk_man->TopUp(); ReserveDestination reservedest(this); if (!reservedest.GetReservedDestination(type, dest, true)) { @@ -3074,11 +3103,12 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: { LOCK(cs_wallet); + std::set<uint256> trusted_parents; for (const auto& walletEntry : mapWallet) { const CWalletTx& wtx = walletEntry.second; - if (!wtx.IsTrusted(locked_chain)) + if (!wtx.IsTrusted(locked_chain, trusted_parents)) continue; if (wtx.IsImmatureCoinBase(locked_chain)) @@ -3229,7 +3259,7 @@ bool ReserveDestination::GetReservedDestination(const OutputType type, CTxDestin if (nIndex == -1) { CKeyPool keypool; - if (!m_spk_man->ReserveKeyFromKeyPool(nIndex, keypool, internal)) { + if (!m_spk_man->GetReservedDestination(type, internal, nIndex, keypool)) { return false; } vchPubKey = keypool.vchPubKey; @@ -3245,7 +3275,7 @@ bool ReserveDestination::GetReservedDestination(const OutputType type, CTxDestin void ReserveDestination::KeepDestination() { if (nIndex != -1) - m_spk_man->KeepKey(nIndex); + m_spk_man->KeepDestination(nIndex); nIndex = -1; vchPubKey = CPubKey(); address = CNoDestination(); @@ -3254,7 +3284,7 @@ void ReserveDestination::KeepDestination() void ReserveDestination::ReturnDestination() { if (nIndex != -1) { - m_spk_man->ReturnKey(nIndex, fInternal, vchPubKey); + m_spk_man->ReturnDestination(nIndex, fInternal, vchPubKey); } nIndex = -1; vchPubKey = CPubKey(); @@ -3600,31 +3630,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, return nullptr; } - bool hd_upgrade = false; - bool split_upgrade = false; - if (walletInstance->CanSupportFeature(FEATURE_HD) && !walletInstance->m_spk_man->IsHDEnabled()) { - walletInstance->WalletLogPrintf("Upgrading wallet to HD\n"); - walletInstance->SetMinVersion(FEATURE_HD); - - // generate a new master key - CPubKey masterPubKey = walletInstance->m_spk_man->GenerateNewSeed(); - walletInstance->m_spk_man->SetHDSeed(masterPubKey); - hd_upgrade = true; - } - // Upgrade to HD chain split if necessary - if (walletInstance->CanSupportFeature(FEATURE_HD_SPLIT)) { - walletInstance->WalletLogPrintf("Upgrading wallet to use HD chain split\n"); - walletInstance->SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); - split_upgrade = FEATURE_HD_SPLIT > prev_version; - } - // Mark all keys currently in the keypool as pre-split - if (split_upgrade) { - walletInstance->MarkPreSplitKeys(); - } - // Regenerate the keypool if upgraded to HD - if (hd_upgrade) { - if (!walletInstance->m_spk_man->TopUpKeyPool()) { - error = _("Unable to generate keys").translated; + if (auto spk_man = walletInstance->m_spk_man.get()) { + std::string error; + if (!spk_man->Upgrade(prev_version, error)) { + chain.initError(error); return nullptr; } } @@ -3637,15 +3646,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->SetWalletFlags(wallet_creation_flags, false); if (!(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { - // generate a new seed - CPubKey seed = walletInstance->m_spk_man->GenerateNewSeed(); - walletInstance->m_spk_man->SetHDSeed(seed); - } - - // Top up the keypool - if (walletInstance->m_spk_man->CanGenerateKeys() && !walletInstance->m_spk_man->TopUpKeyPool()) { - error = _("Unable to generate initial keys").translated; - return nullptr; + if (auto spk_man = walletInstance->m_spk_man.get()) { + if (!spk_man->SetupGeneration()) { + error = _("Unable to generate initial keys").translated; + return nullptr; + } + } } auto locked_chain = chain.lock(); @@ -3655,9 +3661,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, error = strprintf(_("Error loading %s: Private keys can only be disabled during creation").translated, walletFile); return NULL; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - LOCK(walletInstance->cs_KeyStore); - if (!walletInstance->mapKeys.empty() || !walletInstance->mapCryptedKeys.empty()) { - warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys").translated, walletFile)); + if (walletInstance->m_spk_man) { + if (walletInstance->m_spk_man->HavePrivateKeys()) { + warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys").translated, walletFile)); + } } } @@ -3807,8 +3814,13 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) - if (walletInstance->nTimeFirstKey) { - if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(walletInstance->nTimeFirstKey - TIMESTAMP_WINDOW, rescan_height, nullptr)) { + Optional<int64_t> time_first_key; + if (auto spk_man = walletInstance->m_spk_man.get()) { + int64_t time = spk_man->GetTimeFirstKey(); + if (!time_first_key || time < *time_first_key) time_first_key = time; + } + if (time_first_key) { + if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, nullptr)) { rescan_height = *first_block; } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index f3b791441c..6c9b3f40ab 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -476,6 +476,7 @@ public: bool InMempool() const; bool IsTrusted(interfaces::Chain::Lock& locked_chain) const; + bool IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const; int64_t GetTxTime() const; @@ -660,7 +661,10 @@ private: bool SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose); //! Unsets a wallet flag and saves it to disk - void UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag) override; + void UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag); + + //! Unset the blank wallet flag and saves it to disk + void UnsetBlankWalletFlag(WalletBatch& batch) override; /** Interface for accessing chain state. */ interfaces::Chain* m_chain; @@ -989,11 +993,7 @@ public: bool DelAddressBook(const CTxDestination& address); - unsigned int GetKeyPoolSize() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) - { - AssertLockHeld(cs_wallet); - return setInternalKeyPool.size() + setExternalKeyPool.size(); - } + unsigned int GetKeyPoolSize() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! signify that a particular wallet feature is now used. this may change nWalletVersion and nWalletMaxVersion if those are lower void SetMinVersion(enum WalletFeature, WalletBatch* batch_in = nullptr, bool fExplicit = false) override; @@ -1090,7 +1090,7 @@ public: void BlockUntilSyncedToCurrentChain() LOCKS_EXCLUDED(cs_main, cs_wallet); /** set a single wallet flag */ - void SetWalletFlag(uint64_t flags) override; + void SetWalletFlag(uint64_t flags); /** Unsets a single wallet flag */ void UnsetWalletFlag(uint64_t flag); @@ -1128,13 +1128,6 @@ public: LegacyScriptPubKeyMan::WatchOnlySet& setWatchOnly GUARDED_BY(cs_KeyStore) = m_spk_man->setWatchOnly; LegacyScriptPubKeyMan::WatchKeyMap& mapWatchKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapWatchKeys; WalletBatch*& encrypted_batch GUARDED_BY(cs_wallet) = m_spk_man->encrypted_batch; - std::set<int64_t>& setInternalKeyPool GUARDED_BY(cs_wallet) = m_spk_man->setInternalKeyPool; - std::set<int64_t>& setExternalKeyPool GUARDED_BY(cs_wallet) = m_spk_man->setExternalKeyPool; - int64_t& nTimeFirstKey GUARDED_BY(cs_wallet) = m_spk_man->nTimeFirstKey; - std::map<CKeyID, CKeyMetadata>& mapKeyMetadata GUARDED_BY(cs_wallet) = m_spk_man->mapKeyMetadata; - std::map<CScriptID, CKeyMetadata>& m_script_metadata GUARDED_BY(cs_wallet) = m_spk_man->m_script_metadata; - void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(m_spk_man->cs_wallet); m_spk_man->MarkPreSplitKeys(); } - void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(m_spk_man->cs_wallet); m_spk_man->MarkReserveKeysAsUsed(keypool_id); } using CryptedKeyMap = LegacyScriptPubKeyMan::CryptedKeyMap; }; diff --git a/test/README.md b/test/README.md index 11adc11278..24a9389fac 100644 --- a/test/README.md +++ b/test/README.md @@ -254,7 +254,13 @@ Use the `-v` option for verbose output. #### Dependencies -The lint tests require codespell and flake8. To install: `pip3 install codespell flake8`. +| Lint test | Dependency | Version [used by CI](../ci/lint/04_install.sh) | Installation +|-----------|:----------:|:-------------------------------------------:|-------------- +| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.7.8](https://github.com/bitcoin/bitcoin/pull/15257) | `pip3 install flake8==3.7.8` +| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.6.0](https://github.com/bitcoin/bitcoin/pull/15166) | [details...](https://github.com/koalaman/shellcheck#installing) +| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [1.15.0](https://github.com/bitcoin/bitcoin/pull/16186) | `pip3 install codespell==1.15.0` + +Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated. #### Running the tests diff --git a/test/functional/README.md b/test/functional/README.md index a9b83076eb..77a9ce9acb 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -99,6 +99,16 @@ P2PInterface object and override the callback methods. Examples tests are [p2p_unrequested_blocks.py](p2p_unrequested_blocks.py), [p2p_compactblocks.py](p2p_compactblocks.py). +#### Prototyping tests + +The [`TestShell`](test-shell.md) class exposes the BitcoinTestFramework +functionality to interactive Python3 environments and can be used to prototype +tests. This may be especially useful in a REPL environment with session logging +utilities, such as +[IPython](https://ipython.readthedocs.io/en/stable/interactive/reference.html#session-logging-and-restoring). +The logs of such interactive sessions can later be adapted into permanent test +cases. + ### Test framework modules The following are useful modules for test developers. They are located in [test/functional/test_framework/](test_framework). diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py new file mode 100755 index 0000000000..7527bdfb08 --- /dev/null +++ b/test/functional/rpc_dumptxoutset.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the generation of UTXO snapshots using `dumptxoutset`. +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error + +import hashlib +from pathlib import Path + + +class DumptxoutsetTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + """Test a trivial usage of the dumptxoutset RPC command.""" + node = self.nodes[0] + mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1 + node.setmocktime(mocktime) + node.generate(100) + + FILENAME = 'txoutset.dat' + out = node.dumptxoutset(FILENAME) + expected_path = Path(node.datadir) / 'regtest' / FILENAME + + assert expected_path.is_file() + + assert_equal(out['coins_written'], 100) + assert_equal(out['base_height'], 100) + assert_equal(out['path'], str(expected_path)) + # Blockhash should be deterministic based on mocked time. + assert_equal( + out['base_hash'], + '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6') + + with open(str(expected_path), 'rb') as f: + digest = hashlib.sha256(f.read()).hexdigest() + # UTXO snapshot hash should be deterministic based on mocked time. + assert_equal( + digest, 'be032e5f248264ba08e11099ac09dbd001f6f87ffc68bf0f87043d8146d50664') + + # Specifying a path to an existing file will fail. + assert_raises_rpc_error( + -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + +if __name__ == '__main__': + DumptxoutsetTest().main() diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md new file mode 100644 index 0000000000..f6ea9ef682 --- /dev/null +++ b/test/functional/test-shell.md @@ -0,0 +1,186 @@ +Test Shell for Interactive Environments +========================================= + +This document describes how to use the `TestShell` submodule in the functional +test suite. + +The `TestShell` submodule extends the `BitcoinTestFramework` functionality to +external interactive environments for prototyping and educational purposes. Just +like `BitcoinTestFramework`, the `TestShell` allows the user to: + +* Manage regtest bitcoind subprocesses. +* Access RPC interfaces of the underlying bitcoind instances. +* Log events to the functional test logging utility. + +The `TestShell` can be useful in interactive environments where it is necessary +to extend the object lifetime of the underlying `BitcoinTestFramework` between +user inputs. Such environments include the Python3 command line interpreter or +[Jupyter](https://jupyter.org/) notebooks running a Python3 kernel. + +## 1. Requirements + +* Python3 +* `bitcoind` built in the same repository as the `TestShell`. + +## 2. Importing `TestShell` from the Bitcoin Core repository + +We can import the `TestShell` by adding the path of the Bitcoin Core +`test_framework` module to the beginning of the PATH variable, and then +importing the `TestShell` class from the `test_shell` sub-package. + +``` +>>> import sys +>>> sys.path.insert(0, "/path/to/bitcoin/test/functional") +>>> from test_framework.test_shell import TestShell +``` + +The following `TestShell` methods manage the lifetime of the underlying bitcoind +processes and logging utilities. + +* `TestShell.setup()` +* `TestShell.shutdown()` + +The `TestShell` inherits all `BitcoinTestFramework` members and methods, such +as: +* `TestShell.nodes[index].rpc_method()` +* `TestShell.log.info("Custom log message")` + +The following sections demonstrate how to initialize, run, and shut down a +`TestShell` object. + +## 3. Initializing a `TestShell` object + +``` +>>> test = TestShell().setup(num_nodes=2, setup_clean_chain=True) +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Initializing test directory /path/to/bitcoin_func_test_XXXXXXX +``` +The `TestShell` forwards all functional test parameters of the parent +`BitcoinTestFramework` object. The full set of argument keywords which can be +used to initialize the `TestShell` can be found in [section +#6](#custom-testshell-parameters) of this document. + +**Note: Running multiple instances of `TestShell` is not allowed.** Running a +single process also ensures that logging remains consolidated in the same +temporary folder. If you need more bitcoind nodes than set by default (1), +simply increase the `num_nodes` parameter during setup. + +``` +>>> test2 = TestShell().setup() +TestShell is already running! +``` + +## 4. Interacting with the `TestShell` + +Unlike the `BitcoinTestFramework` class, the `TestShell` keeps the underlying +Bitcoind subprocesses (nodes) and logging utilities running until the user +explicitly shuts down the `TestShell` object. + +During the time between the `setup` and `shutdown` calls, all `bitcoind` node +processes and `BitcoinTestFramework` convenience methods can be accessed +interactively. + +**Example: Mining a regtest chain** + +By default, the `TestShell` nodes are initialized with a clean chain. This means +that each node of the `TestShell` is initialized with a block height of 0. + +``` +>>> test.nodes[0].getblockchaininfo()["blocks"] +0 +``` + +We now let the first node generate 101 regtest blocks, and direct the coinbase +rewards to a wallet address owned by the mining node. + +``` +>>> address = test.nodes[0].getnewaddress() +>>> test.nodes[0].generatetoaddress(101, address) +['2b98dd0044aae6f1cca7f88a0acf366a4bfe053c7f7b00da3c0d115f03d67efb', ... +``` +Since the two nodes are both initialized by default to establish an outbound +connection to each other during `setup`, the second node's chain will include +the mined blocks as soon as they propagate. + +``` +>>> test.nodes[1].getblockchaininfo()["blocks"] +101 +``` +The block rewards from the first block are now spendable by the wallet of the +first node. + +``` +>>> test.nodes[0].getbalance() +Decimal('50.00000000') +``` + +We can also log custom events to the logger. + +``` +>>> test.nodes[0].log.info("Successfully mined regtest chain!") +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework.node0 (INFO): Successfully mined regtest chain! +``` + +**Note: Please also consider the functional test +[readme](../test/functional/README.md), which provides an overview of the +test-framework**. Modules such as +[key.py](../test/functional/test_framework/key.py), +[script.py](../test/functional/test_framework/script.py) and +[messages.py](../test/functional/test_framework/messages.py) are particularly +useful in constructing objects which can be passed to the bitcoind nodes managed +by a running `TestShell` object. + +## 5. Shutting the `TestShell` down + +Shutting down the `TestShell` will safely tear down all running bitcoind +instances and remove all temporary data and logging directories. + +``` +>>> test.shutdown() +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Stopping nodes +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Cleaning up /path/to/bitcoin_func_test_XXXXXXX on exit +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Tests successful +``` +To prevent the logs from being removed after a shutdown, simply set the +`TestShell.options.nocleanup` member to `True`. +``` +>>> test.options.nocleanup = True +>>> test.shutdown() +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Stopping nodes +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Not cleaning up dir /path/to/bitcoin_func_test_XXXXXXX on exit +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Tests successful +``` + +The following utility consolidates logs from the bitcoind nodes and the +underlying `BitcoinTestFramework`: + +* `/path/to/bitcoin/test/functional/combine_logs.py + '/path/to/bitcoin_func_test_XXXXXXX'` + +## 6. Custom `TestShell` parameters + +The `TestShell` object initializes with the default settings inherited from the +`BitcoinTestFramework` class. The user can override these in +`TestShell.setup(key=value)`. + +**Note:** `TestShell.reset()` will reset test parameters to default values and +can be called after the TestShell is shut down. + +| Test parameter key | Default Value | Description | +|---|---|---| +| `bind_to_localhost_only` | `True` | Binds bitcoind RPC services to `127.0.0.1` if set to `True`.| +| `cachedir` | `"/path/to/bitcoin/test/cache"` | Sets the bitcoind datadir directory. | +| `chain` | `"regtest"` | Sets the chain-type for the underlying test bitcoind processes. | +| `configfile` | `"/path/to/bitcoin/test/config.ini"` | Sets the location of the test framework config file. | +| `coveragedir` | `None` | Records bitcoind RPC test coverage into this directory if set. | +| `loglevel` | `INFO` | Logs events at this level and higher. Can be set to `DEBUG`, `INFO`, `WARNING`, `ERROR` or `CRITICAL`. | +| `nocleanup` | `False` | Cleans up temporary test directory if set to `True` during `shutdown`. | +| `noshutdown` | `False` | Does not stop bitcoind instances after `shutdown` if set to `True`. | +| `num_nodes` | `1` | Sets the number of initialized bitcoind processes. | +| `perf` | False | Profiles running nodes with `perf` for the duration of the test if set to `True`. | +| `rpc_timeout` | `60` | Sets the RPC server timeout for the underlying bitcoind processes. | +| `setup_clean_chain` | `False` | Initializes an empty blockchain by default. A 199-block-long chain is initialized if set to `True`. | +| `randomseed` | Random Integer | `TestShell.options.randomseed` is a member of `TestShell` which can be accessed during a test to seed a random generator. User can override default with a constant value for reproducible test runs. | +| `supports_cli` | `False` | Whether the bitcoin-cli utility is compiled and available for the test. | +| `tmpdir` | `"/var/folders/.../"` | Sets directory for test logs. Will be deleted upon a successful test run unless `nocleanup` is set to `True` | +| `trace_rpc` | `False` | Logs all RPC calls if set to `True`. | +| `usecli` | `False` | Uses the bitcoin-cli interface for all bitcoind commands instead of directly calling the RPC server. Requires `supports_cli`. | diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index f95c158a68..a9e669fea9 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -478,7 +478,8 @@ class NetworkThread(threading.Thread): wait_until(lambda: not self.network_event_loop.is_running(), timeout=timeout) self.network_event_loop.close() self.join(timeout) - + # Safe to remove event loop. + NetworkThread.network_event_loop = None class P2PDataStore(P2PInterface): """A P2P data store class. diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 780aa5fe03..c56c0d06ff 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -99,12 +99,39 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.supports_cli = False self.bind_to_localhost_only = True self.set_test_params() - - assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" + self.parse_args() def main(self): """Main function. This should not be overridden by the subclass test scripts.""" + assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" + + try: + self.setup() + self.run_test() + except JSONRPCException: + self.log.exception("JSONRPC error") + self.success = TestStatus.FAILED + except SkipTest as e: + self.log.warning("Test Skipped: %s" % e.message) + self.success = TestStatus.SKIPPED + except AssertionError: + self.log.exception("Assertion failed") + self.success = TestStatus.FAILED + except KeyError: + self.log.exception("Key error") + self.success = TestStatus.FAILED + except Exception: + self.log.exception("Unexpected exception caught during testing") + self.success = TestStatus.FAILED + except KeyboardInterrupt: + self.log.warning("Exiting after keyboard interrupt") + self.success = TestStatus.FAILED + finally: + exit_code = self.shutdown() + sys.exit(exit_code) + + def parse_args(self): parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") @@ -135,6 +162,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.add_options(parser) self.options = parser.parse_args() + def setup(self): + """Call this method to start up the test framework object with options set.""" + PortSeed.n = self.options.port_seed check_json_precision() @@ -181,33 +211,20 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.network_thread = NetworkThread() self.network_thread.start() - success = TestStatus.FAILED + if self.options.usecli: + if not self.supports_cli: + raise SkipTest("--usecli specified but test does not support using CLI") + self.skip_if_no_cli() + self.skip_test_if_missing_module() + self.setup_chain() + self.setup_network() - try: - if self.options.usecli: - if not self.supports_cli: - raise SkipTest("--usecli specified but test does not support using CLI") - self.skip_if_no_cli() - self.skip_test_if_missing_module() - self.setup_chain() - self.setup_network() - self.run_test() - success = TestStatus.PASSED - except JSONRPCException: - self.log.exception("JSONRPC error") - except SkipTest as e: - self.log.warning("Test Skipped: %s" % e.message) - success = TestStatus.SKIPPED - except AssertionError: - self.log.exception("Assertion failed") - except KeyError: - self.log.exception("Key error") - except Exception: - self.log.exception("Unexpected exception caught during testing") - except KeyboardInterrupt: - self.log.warning("Exiting after keyboard interrupt") + self.success = TestStatus.PASSED - if success == TestStatus.FAILED and self.options.pdbonfailure: + def shutdown(self): + """Call this method to shut down the test framework object.""" + + if self.success == TestStatus.FAILED and self.options.pdbonfailure: print("Testcase failed. Attaching python debugger. Enter ? for help") pdb.set_trace() @@ -225,7 +242,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): should_clean_up = ( not self.options.nocleanup and not self.options.noshutdown and - success != TestStatus.FAILED and + self.success != TestStatus.FAILED and not self.options.perf ) if should_clean_up: @@ -238,20 +255,33 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.log.warning("Not cleaning up dir {}".format(self.options.tmpdir)) cleanup_tree_on_exit = False - if success == TestStatus.PASSED: + if self.success == TestStatus.PASSED: self.log.info("Tests successful") exit_code = TEST_EXIT_PASSED - elif success == TestStatus.SKIPPED: + elif self.success == TestStatus.SKIPPED: self.log.info("Test skipped") exit_code = TEST_EXIT_SKIPPED else: self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) exit_code = TEST_EXIT_FAILED - logging.shutdown() + # Logging.shutdown will not remove stream- and filehandlers, so we must + # do it explicitly. Handlers are removed so the next test run can apply + # different log handler settings. + # See: https://docs.python.org/3/library/logging.html#logging.shutdown + for h in list(self.log.handlers): + h.flush() + h.close() + self.log.removeHandler(h) + rpc_logger = logging.getLogger("BitcoinRPC") + for h in list(rpc_logger.handlers): + h.flush() + rpc_logger.removeHandler(h) if cleanup_tree_on_exit: shutil.rmtree(self.options.tmpdir) - sys.exit(exit_code) + + self.nodes.clear() + return exit_code # Methods to override in subclass test scripts. def set_test_params(self): diff --git a/test/functional/test_framework/test_shell.py b/test/functional/test_framework/test_shell.py new file mode 100644 index 0000000000..26df128f1f --- /dev/null +++ b/test/functional/test_framework/test_shell.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import BitcoinTestFramework + +class TestShell: + """Wrapper Class for BitcoinTestFramework. + + The TestShell class extends the BitcoinTestFramework + rpc & daemon process management functionality to external + python environments. + + It is a singleton class, which ensures that users only + start a single TestShell at a time.""" + + class __TestShell(BitcoinTestFramework): + def set_test_params(self): + pass + + def run_test(self): + pass + + def setup(self, **kwargs): + if self.running: + print("TestShell is already running!") + return + + # Num_nodes parameter must be set + # by BitcoinTestFramework child class. + self.num_nodes = 1 + + # User parameters override default values. + for key, value in kwargs.items(): + if hasattr(self, key): + setattr(self, key, value) + elif hasattr(self.options, key): + setattr(self.options, key, value) + else: + raise KeyError(key + " not a valid parameter key!") + + super().setup() + self.running = True + return self + + def shutdown(self): + if not self.running: + print("TestShell is not running!") + else: + super().shutdown() + self.running = False + + def reset(self): + if self.running: + print("Shutdown TestShell before resetting!") + else: + self.num_nodes = None + super().__init__() + + instance = None + + def __new__(cls): + # This implementation enforces singleton pattern, and will return the + # previously initialized instance if available + if not TestShell.instance: + TestShell.instance = TestShell.__TestShell() + TestShell.instance.running = False + return TestShell.instance + + def __getattr__(self, name): + return getattr(self.instance, name) + + def __setattr__(self, name, value): + return setattr(self.instance, name, value) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6665d2e7f5..9b4a5d2030 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -191,6 +191,7 @@ BASE_SCRIPTS = [ 'rpc_uptime.py', 'wallet_resendwallettransactions.py', 'wallet_fallbackfee.py', + 'rpc_dumptxoutset.py', 'feature_minchainwork.py', 'rpc_getblockstats.py', 'wallet_create_tx.py', diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index c50dcd987a..a5f9a047ed 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -109,13 +109,51 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs") + # Before `test_balance()`, we have had two nodes with a balance of 50 + # each and then we: + # + # 1) Sent 40 from node A to node B with fee 0.01 + # 2) Sent 60 from node B to node A with fee 0.01 + # + # Then we check the balances: + # + # 1) As is + # 2) With transaction 2 from above with 2x the fee + # + # Prior to #16766, in this situation, the node would immediately report + # a balance of 30 on node B as unconfirmed and trusted. + # + # After #16766, we show that balance as unconfirmed. + # + # The balance is indeed "trusted" and "confirmed" insofar as removing + # the mempool transactions would return at least that much money. But + # the algorithm after #16766 marks it as unconfirmed because the 'taint' + # tracking of transaction trust for summing balances doesn't consider + # which inputs belong to a user. In this case, the change output in + # question could be "destroyed" by replace the 1st transaction above. + # + # The post #16766 behavior is correct; we shouldn't be treating those + # funds as confirmed. If you want to rely on that specific UTXO existing + # which has given you that balance, you cannot, as a third party + # spending the other input would destroy that unconfirmed. + # + # For example, if the test transactions were: + # + # 1) Sent 40 from node A to node B with fee 0.01 + # 2) Sent 10 from node B to node A with fee 0.01 + # + # Then our node would report a confirmed balance of 40 + 50 - 10 = 80 + # BTC, which is more than would be available if transaction 1 were + # replaced. + + def test_balances(*, fee_node_1=0): # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send - assert_equal(self.nodes[1].getbalance(), Decimal('30') - fee_node_1) # change from node 1's send + assert_equal(self.nodes[1].getbalance(), Decimal('0')) # node 1's send had an unsafe input # Same with minconf=0 assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99')) - assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('30') - fee_node_1) + assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('0')) # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago # TODO: fix getbalance tracking of coin spentness depth assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0')) @@ -125,9 +163,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('60')) assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) - assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) # Doesn't include output of node 0's send since it was spent - assert_equal(self.nodes[1].getbalances()['mine']['untrusted_pending'], Decimal('0')) - assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0')) + assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('30') - fee_node_1) # Doesn't include output of node 0's send since it was spent + assert_equal(self.nodes[1].getbalances()['mine']['untrusted_pending'], Decimal('30') - fee_node_1) + assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('30') - fee_node_1) test_balances(fee_node_1=Decimal('0.01')) diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 0948d47653..95d51adebb 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -38,7 +38,7 @@ class BumpFeeTest(BitcoinTestFramework): "-walletrbf={}".format(i), "-mintxfee=0.00002", "-deprecatedrpc=totalFee", - "-addresstype=p2sh-segwit", # TODO update constants in test and remove + "-addresstype=bech32", ] for i in range(self.num_nodes)] def skip_test_if_missing_module(self): @@ -246,10 +246,8 @@ def test_dust_to_fee(rbf_node, dest_address): # the bumped tx sets fee=49,900, but it converts to 50,000 rbfid = spend_one_input(rbf_node, dest_address) fulltx = rbf_node.getrawtransaction(rbfid, 1) - # (32-byte p2sh-pwpkh output size + 148 p2pkh spend estimate) * 10k(discard_rate) / 1000 = 1800 - # P2SH outputs are slightly "over-discarding" due to the IsDust calculation assuming it will - # be spent as a P2PKH. - bumped_tx = rbf_node.bumpfee(rbfid, {"totalFee": 50000 - 1800}) + # (31-vbyte p2wpkh output size + 67-vbyte p2wpkh spend estimate) * 10k(discard_rate) / 1000 = 980 + bumped_tx = rbf_node.bumpfee(rbfid, {"totalFee": 50000 - 980}) full_bumped_tx = rbf_node.getrawtransaction(bumped_tx["txid"], 1) assert_equal(bumped_tx["fee"], Decimal("0.00050000")) assert_equal(len(fulltx["vout"]), 2) @@ -272,7 +270,9 @@ def test_settxfee(rbf_node, dest_address): def test_maxtxfee_fails(test, rbf_node, dest_address): - test.restart_node(1, ['-maxtxfee=0.00003'] + test.extra_args[1]) + # size of bumped transaction (p2wpkh, 1 input, 2 outputs): 141 vbytes + # expected bumping feerate of 20 sats/vbyte => 141*20 sats = 0.00002820 btc + test.restart_node(1, ['-maxtxfee=0.000025'] + test.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) rbfid = spend_one_input(rbf_node, dest_address) assert_raises_rpc_error(-4, "Unable to create transaction: Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 4aeb393255..455e89e310 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -5,6 +5,7 @@ """Test the listsincelast RPC.""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.messages import BIP125_SEQUENCE_NUMBER from test_framework.util import ( assert_array_result, assert_equal, @@ -12,6 +13,7 @@ from test_framework.util import ( connect_nodes, ) +from decimal import Decimal class ListSinceBlockTest(BitcoinTestFramework): def set_test_params(self): @@ -33,6 +35,7 @@ class ListSinceBlockTest(BitcoinTestFramework): self.test_reorg() self.test_double_spend() self.test_double_send() + self.double_spends_filtered() def test_no_blockhash(self): txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) @@ -291,5 +294,51 @@ class ListSinceBlockTest(BitcoinTestFramework): if tx['txid'] == txid1: assert_equal(tx['confirmations'], 2) + def double_spends_filtered(self): + ''' + `listsinceblock` was returning conflicted transactions even if they + occurred before the specified cutoff blockhash + ''' + spending_node = self.nodes[2] + dest_address = spending_node.getnewaddress() + + tx_input = dict( + sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in spending_node.listunspent())) + rawtx = spending_node.createrawtransaction( + [tx_input], {dest_address: tx_input["amount"] - Decimal("0.00051000"), + spending_node.getrawchangeaddress(): Decimal("0.00050000")}) + signedtx = spending_node.signrawtransactionwithwallet(rawtx) + orig_tx_id = spending_node.sendrawtransaction(signedtx["hex"]) + original_tx = spending_node.gettransaction(orig_tx_id) + + double_tx = spending_node.bumpfee(orig_tx_id) + + # check that both transactions exist + block_hash = spending_node.listsinceblock( + spending_node.getblockhash(spending_node.getblockcount())) + original_found = False + double_found = False + for tx in block_hash['transactions']: + if tx['txid'] == original_tx['txid']: + original_found = True + if tx['txid'] == double_tx['txid']: + double_found = True + assert_equal(original_found, True) + assert_equal(double_found, True) + + lastblockhash = spending_node.generate(1)[0] + + # check that neither transaction exists + block_hash = spending_node.listsinceblock(lastblockhash) + original_found = False + double_found = False + for tx in block_hash['transactions']: + if tx['txid'] == original_tx['txid']: + original_found = True + if tx['txid'] == double_tx['txid']: + double_found = True + assert_equal(original_found, False) + assert_equal(double_found, False) + if __name__ == '__main__': ListSinceBlockTest().main() diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh index 5bbcae79eb..a4c6f0a8d4 100755 --- a/test/lint/lint-assertions.sh +++ b/test/lint/lint-assertions.sh @@ -20,4 +20,15 @@ if [[ ${OUTPUT} != "" ]]; then EXIT_CODE=1 fi +# Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it +# is undesirable to crash the whole program. See: src/util/check.h +# src/rpc/server.cpp is excluded from this check since it's mostly meta-code. +OUTPUT=$(git grep -nE 'assert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") +if [[ ${OUTPUT} != "" ]]; then + echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code." + echo + echo "${OUTPUT}" + EXIT_CODE=1 +fi + exit ${EXIT_CODE} diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 3c82ec19e3..2b1a1d8fbc 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -82,10 +82,10 @@ enabled=( ) if ! command -v flake8 > /dev/null; then - echo "Skipping Python linting since flake8 is not installed. Install by running \"pip3 install flake8\"" + echo "Skipping Python linting since flake8 is not installed." exit 0 elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then - echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8 by running \"pip3 install flake8\"" + echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8." exit 0 fi diff --git a/test/lint/lint-spelling.ignore-words.txt b/test/lint/lint-spelling.ignore-words.txt index b08837c1d4..576ae94098 100644 --- a/test/lint/lint-spelling.ignore-words.txt +++ b/test/lint/lint-spelling.ignore-words.txt @@ -1,9 +1,7 @@ -cas hights mor mut objext -unselect useable wit unparseable @@ -13,3 +11,4 @@ errorstring keyserver homogenous setban +hist diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh index e70b73e1cc..251a94f7ff 100755 --- a/test/lint/lint-spelling.sh +++ b/test/lint/lint-spelling.sh @@ -15,6 +15,6 @@ if ! command -v codespell > /dev/null; then fi IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt -if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/qt/locale/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/"); then +if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/"); then echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" fi |