diff options
150 files changed, 5248 insertions, 1306 deletions
diff --git a/.travis.yml b/.travis.yml index ea589b4059..63e502cc3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -128,7 +128,7 @@ jobs: FILE_ENV="./ci/test/00_setup_env_i686_centos.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [bionic] [previous releases, uses qt5 dev package and some depends packages] [unsigned char]' + name: 'x86_64 Linux [GOAL: install] [bionic] [C++17, previous releases, uses qt5 dev package and some depends packages] [unsigned char]' env: >- FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" @@ -25,7 +25,7 @@ information or see https://opensource.org/licenses/MIT. Development Process ------------------- -The `master` branch is regularly built and tested, but is not guaranteed to be +The `master` branch is regularly built (see doc/build-*.md for instructions) and tested, but is not guaranteed to be completely stable. [Tags](https://github.com/bitcoin/bitcoin/tags) are created regularly to indicate new official, stable release versions of Bitcoin Core. diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 index f147cee3b1..43087b2e68 100644 --- a/build-aux/m4/ax_cxx_compile_stdcxx.m4 +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html # =========================================================================== # # SYNOPSIS @@ -33,21 +33,23 @@ # Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com> # Copyright (c) 2015 Paul Norman <penorman@mac.com> # Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> +# Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com> +# Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com> # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 4 +#serial 11 dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro dnl (serial version number 13). AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl - m4_if([$1], [11], [], - [$1], [14], [], - [$1], [17], [m4_fatal([support for C++17 not yet implemented in AX_CXX_COMPILE_STDCXX])], + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl m4_if([$2], [], [], [$2], [ext], [], @@ -57,26 +59,13 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], [$3], [optional], [ax_cxx_compile_cxx$1_required=false], [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) - m4_if([$4], [], [ax_cxx_compile_cxx$1_try_default=true], - [$4], [default], [ax_cxx_compile_cxx$1_try_default=true], - [$4], [nodefault], [ax_cxx_compile_cxx$1_try_default=false], - [m4_fatal([invalid fourth argument `$4' to AX_CXX_COMPILE_STDCXX])]) AC_LANG_PUSH([C++])dnl ac_success=no - m4_if([$4], [nodefault], [], [dnl - AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, - ax_cv_cxx_compile_cxx$1, - [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], - [ax_cv_cxx_compile_cxx$1=yes], - [ax_cv_cxx_compile_cxx$1=no])]) - if test x$ax_cv_cxx_compile_cxx$1 = xyes; then - ac_success=yes - fi]) - m4_if([$2], [noext], [], [dnl if test x$ac_success = xno; then - for switch in -std=gnu++$1 -std=gnu++0x; do + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, $cachevar, @@ -102,22 +91,27 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl dnl HP's aCC needs +std=c++11 according to: dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf dnl Cray's crayCC needs "-h std=c++11" - for switch in -std=c++$1 -std=c++0x +std=c++$1 "-h std=c++$1"; do - cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) - AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, - $cachevar, - [ac_save_CXX="$CXX" - CXX="$CXX $switch" - AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], - [eval $cachevar=yes], - [eval $cachevar=no]) - CXX="$ac_save_CXX"]) - if eval test x\$$cachevar = xyes; then - CXX="$CXX $switch" - if test -n "$CXXCPP" ; then - CXXCPP="$CXXCPP $switch" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break fi - ac_success=yes + done + if test x$ac_success = xyes; then break fi done @@ -154,6 +148,11 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 ) +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) dnl Tests for new features in C++11 @@ -191,11 +190,13 @@ namespace cxx11 struct Base { + virtual ~Base() {} virtual void f() {} }; struct Derived : public Base { + virtual ~Derived() override {} virtual void f() override {} }; @@ -524,7 +525,7 @@ namespace cxx14 } - namespace test_digit_seperators + namespace test_digit_separators { constexpr auto ten_million = 100'000'000; @@ -566,3 +567,385 @@ namespace cxx14 #endif // __cplusplus >= 201402L ]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include <initializer_list> +#include <utility> +#include <type_traits> + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template<typename... Args> + int multiply(Args... args) + { + return (args * ... * 1); + } + + template<typename... Args> + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value); + static_assert(std::is_same<int, decltype(bar)>::value); + } + + namespace test_typename_in_template_template_parameter + { + + template<template<typename> typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template <bool cond> + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template <typename T1, typename T2> + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template <auto n> + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair<int, int> pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair<int, int>& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template<typename T> + Bad + f(T*, T*); + + template<typename T1, typename T2> + Good + f(T1*, T2*); + + static_assert (std::is_same_v<Good, decltype(f(g1, g2))>); + + } + + namespace test_inline_variables + { + + template<class T> void f(T) + {} + + template<class T> inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index e0e555bbd1..dae61c5e34 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 # The root dir. # The ci system copies this folder. -# This is where the build is done (depends and dist). +# This is where the depends build is done. BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) export BASE_ROOT_DIR @@ -49,12 +49,14 @@ export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} # The depends dir. # This folder exists on the ci host and ci guest. Changes are propagated back and forth. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} -# Folder where the build is done (bin and lib). +# Folder where the build result is put (bin and lib). export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} +# Folder where the build is done (dist and out-of-tree build). +export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH -export CI_RETRY_EXE=${CI_RETRY_EXE:retry} +export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"} diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index 67fbd1438a..982e38daee 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -8,10 +8,9 @@ export LC_ALL=C.UTF-8 export HOST=x86_64-apple-darwin16 export PIP_PACKAGES="zmq" -export RUN_UNIT_TESTS=true -export RUN_FUNCTIONAL_TESTS=false export GOAL="install" export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" +export TEST_RUNNER_EXTRA="wallet_disable" # Only run wallet_disable as a smoke test, see https://github.com/bitcoin/bitcoin/pull/17240#issuecomment-546022121 why the other tests are disabled # Run without depends export NO_DEPENDS=1 export OSX_SDK="" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 87c8b70998..4dedea23bc 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -14,4 +14,4 @@ export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" export TEST_PREVIOUS_RELEASES=true -export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-c++17 --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index d5b38a1d88..153a781b0a 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -13,7 +13,7 @@ export NO_DEPENDS=1 if [[ "${TRAVIS}" == "true" && "${TRAVIS_REPO_SLUG}" != "bitcoin/bitcoin" ]]; then export TEST_RUNNER_EXTRA="wallet_disable" # Only run wallet_disable as a smoke test to not hit the 50 min travis time limit else - export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 + export TEST_RUNNER_EXTRA="--exclude rpc_bind --factor=2" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 fi export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 32eaaf15bb..5dbf1b82f1 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -18,7 +18,7 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then ${CI_RETRY_EXE} pip3 install $PIP_PACKAGES fi -mkdir -p "${BASE_SCRATCH_DIR}" +# Create folders that are mounted into the docker mkdir -p "${CCACHE_DIR}" mkdir -p "${PREVIOUS_RELEASES_DIR}" @@ -58,6 +58,7 @@ else bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" } fi +export -f DOCKER_EXEC if [ -n "$DPKG_ADD_ARCH" ]; then DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 17853e75dd..ab9d673101 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -37,6 +37,6 @@ if [ -z "$NO_DEPENDS" ]; then fi if [ "$TEST_PREVIOUS_RELEASES" = "true" ]; then BEGIN_FOLD previous-versions - DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" v0.17.1 v0.18.1 v0.19.0.1 + DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" v0.15.2 v0.16.3 v0.17.1 v0.18.1 v0.19.0.1 END_FOLD fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index f568dbe37d..b68cd9d3f8 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -17,20 +17,18 @@ else fi END_FOLD -DOCKER_EXEC mkdir -p build -export P_CI_DIR="$P_CI_DIR/build" +DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}" +export P_CI_DIR="${BASE_BUILD_DIR}" BEGIN_FOLD configure -DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) +DOCKER_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) END_FOLD BEGIN_FOLD distdir -# Create folder on host and docker, so that `cd` works -mkdir -p "bitcoin-$HOST" DOCKER_EXEC make distdir VERSION=$HOST END_FOLD -export P_CI_DIR="$P_CI_DIR/bitcoin-$HOST" +export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST" BEGIN_FOLD configure DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 0f03d1770e..c8542862f2 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -31,7 +31,7 @@ fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then BEGIN_FOLD unit-tests-seq - DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_ROOT_DIR}/build/bitcoin-*/src/test/test_bitcoin" --catch_system_errors=no -l test_suite + DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite END_FOLD fi diff --git a/configure.ac b/configure.ac index 546d00de1b..6046b05611 100644 --- a/configure.ac +++ b/configure.ac @@ -61,8 +61,20 @@ case $host in lt_cv_deplibs_check_method="pass_all" ;; esac -dnl Require C++11 compiler (no GNU extensions) -AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault]) + +AC_ARG_ENABLE([c++17], + [AS_HELP_STRING([--enable-c++17], + [enable compilation in c++17 mode (disabled by default)])], + [use_cxx17=$enableval], + [use_cxx17=no]) + +dnl Require C++11 or C++17 compiler (no GNU extensions) +if test "x$use_cxx17" = xyes; then + AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) +else + AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory]) +fi + dnl Check if -latomic is required for <std::atomic> CHECK_ATOMIC @@ -337,6 +349,7 @@ fi if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wextra],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wgnu],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wswitch],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wswitch"],,[[$CXXFLAG_WERROR]]) @@ -757,7 +770,9 @@ if test x$use_hardening != xno; then esac fi -dnl this flag screws up non-darwin gcc even when the check fails. special-case it. +dnl These flags are specific to ld64, and may cause issues with other linkers. +dnl For example: GNU ld will intepret -dead_strip as -de and then try and use +dnl "ad_strip" as the symbol for the entry point. if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"]) AX_CHECK_LINK_FLAG([[-Wl,-dead_strip_dylibs]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"]) @@ -844,6 +859,22 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]], [ AC_MSG_RESULT(no)] ) +dnl Check for posix_fallocate +AC_MSG_CHECKING(for posix_fallocate) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + // same as in src/util/system.cpp + #ifdef __linux__ + #ifdef _POSIX_C_SOURCE + #undef _POSIX_C_SOURCE + #endif + #define _POSIX_C_SOURCE 200112L + #endif // __linux__ + #include <fcntl.h>]], + [[ int f = posix_fallocate(0, 0, 0); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_POSIX_FALLOCATE, 1,[Define this symbol if you have posix_fallocate]) ], + [ AC_MSG_RESULT(no)] +) + AC_MSG_CHECKING([for visibility attribute]) AC_LINK_IFELSE([AC_LANG_SOURCE([ int foo_def( void ) __attribute__((visibility("default"))); diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 65a80b4102..9444271bdc 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -158,6 +158,17 @@ def check_PE_HIGH_ENTROPY_VA(executable): reqbits = 0 return (bits & reqbits) == reqbits +def check_PE_RELOC_SECTION(executable) -> bool: + '''Check for a reloc section. This is required for functional ASLR.''' + p = subprocess.Popen([OBJDUMP_CMD, '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + for line in stdout.splitlines(): + if '.reloc' in line: + return True + return False + def check_PE_NX(executable): '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' (arch,bits) = get_PE_dll_characteristics(executable) @@ -247,7 +258,8 @@ CHECKS = { 'PE': [ ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), - ('NX', check_PE_NX) + ('NX', check_PE_NX), + ('RELOC_SECTION', check_PE_RELOC_SECTION) ], 'MACHO': [ ('PIE', check_MACHO_PIE), diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index d09f1d0064..ea70b27941 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -49,13 +49,15 @@ class TestSecurityChecks(unittest.TestCase): cc = 'x86_64-w64-mingw32-gcc' write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va']), - (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA NX')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va']), - (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--no-high-entropy-va']), - (1, executable+': failed HIGH_ENTROPY_VA')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed HIGH_ENTROPY_VA RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), (0, '')) def test_MACHO(self): diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 6f79d10e68..f421372e10 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -140,18 +140,12 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=$(echo bitcoin-*.tar.gz) - DISTNAME=${SOURCEDIST/%.tar.gz} - - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. + # shellcheck source=contrib/gitian-descriptors/make_git_archive + source contrib/gitian-descriptors/make_git_archive ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} if [ "${i}" = "riscv64-linux-gnu" ]; then @@ -165,13 +159,9 @@ script: | cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ + tar -xf $GIT_ARCHIVE + ./autogen.sh CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security @@ -183,12 +173,9 @@ script: | rm -rf ${DISTNAME}/lib/pkgconfig find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - cp ../../README.md ${DISTNAME}/ + cp ../README.md ${DISTNAME}/ find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ rm -rf distsrc-${i} done - - mkdir -p ${OUTDIR}/src - git archive --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 37f2a534b8..82f8f194fc 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -103,31 +103,21 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=$(echo bitcoin-*.tar.gz) - DISTNAME=${SOURCEDIST/%.tar.gz} - - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. + # shellcheck source=contrib/gitian-descriptors/make_git_archive + source contrib/gitian-descriptors/make_git_archive ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ + tar -xf $GIT_ARCHIVE + ./autogen.sh CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security @@ -160,7 +150,4 @@ script: | cd ../../ done - mkdir -p ${OUTDIR}/src - git archive --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD - mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 0cc1adc557..54ad68a2a3 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -76,13 +76,11 @@ script: | function create_per-host_compiler_wrapper { # -posix variant is required for c++11 threading. for i in $HOSTS; do - mkdir -p ${WRAP_DIR}/${i} for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done @@ -110,38 +108,28 @@ script: | create_per-host_compiler_wrapper "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the release tarball using (arbitrarily) the first host - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ - make dist - SOURCEDIST=$(echo bitcoin-*.tar.gz) - DISTNAME=${SOURCEDIST/%.tar.gz} - - # Workaround for tarball not building with the bare tag version (prep) - make -C src obj/build.h + # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. + # shellcheck source=contrib/gitian-descriptors/make_git_archive + source contrib/gitian-descriptors/make_git_archive ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf ../$SOURCEDIST - - # Workaround for tarball not building with the bare tag version - echo '#!/bin/true' >share/genbuild.sh - mkdir src/obj - cp ../src/obj/build.h src/obj/ + tar -xf $GIT_ARCHIVE + ./autogen.sh CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make ${MAKEOPTS} -C src check-symbols make deploy make install DESTDIR=${INSTALLPATH} - cp -f --target-directory="${OUTDIR}" ./bitcoin-*-setup-unsigned.exe + cp -f ./bitcoin-*-win64-setup-unsigned.exe ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete @@ -156,11 +144,8 @@ script: | rm -rf distsrc-${i} done - mkdir -p ${OUTDIR}/src - git archive --output=${OUTDIR}/src/${DISTNAME}.tar.gz HEAD - cp -rf contrib/windeploy $BUILD_DIR cd $BUILD_DIR/windeploy mkdir unsigned - cp $OUTDIR/bitcoin-*setup-unsigned.exe unsigned/ + cp ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe unsigned/ find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz diff --git a/contrib/gitian-descriptors/make_git_archive b/contrib/gitian-descriptors/make_git_archive new file mode 100755 index 0000000000..d922c94c60 --- /dev/null +++ b/contrib/gitian-descriptors/make_git_archive @@ -0,0 +1,20 @@ +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# A helper script to be sourced into the gitian descriptors + +mkdir -p ${OUTDIR}/src +RECENT_TAG=$(git describe --abbrev=0 HEAD) +if [ $RECENT_TAG = $(git describe HEAD) ]; then + if [[ $RECENT_TAG == v* ]]; then + VERSION=${RECENT_TAG:1} + else + VERSION=$RECENT_TAG + fi +else + VERSION=$(git rev-parse --short HEAD) +fi +DISTNAME=bitcoin-${VERSION} +GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" +git archive --output=$GIT_ARCHIVE HEAD diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 744b8ee70f..f232bb62c2 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -184,16 +184,3 @@ ... fun:_ZN5BCLog6Logger12StartLoggingEv } -{ - Suppress BCLog::Logger::StartLogging() still reachable memory warning - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - ... - fun:_ZN5BCLog6Logger12StartLoggingEv -} -{ - Suppress rest_blockhash_by_height Conditional jump or move depends on uninitialised value(s) - Memcheck:Cond - fun:_ZL24rest_blockhash_by_heightP11HTTPRequestRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE -} diff --git a/doc/REST-interface.md b/doc/REST-interface.md index a63391e01a..11e9c90f49 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -30,7 +30,7 @@ To query for a confirmed transaction, enable the transaction index via "txindex= Given a block hash: returns a block, in binary, hex-encoded binary or JSON formats. Responds with 404 if the block doesn't exist. -The HTTP request and response are both handled entirely in-memory, thus making maximum memory usage at least 2.66MB (1 MB max block, plus hex encoding) per request. +The HTTP request and response are both handled entirely in-memory. With the /notxdetails/ option JSON response will only contain the transaction hash instead of the complete transaction details. The option only affects the JSON response. diff --git a/share/genbuild.sh b/share/genbuild.sh index 197787d5e0..81fa2ed5d0 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -18,13 +18,9 @@ else exit 1 fi -git_check_in_repo() { - ! { git status --porcelain -uall --ignored "$@" 2>/dev/null || echo '??'; } | grep -q '?' -} - DESC="" SUFFIX="" -if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ] && git_check_in_repo share/genbuild.sh; then +if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ]; then # clean 'dirty' status of touched files that haven't been modified git diff >/dev/null 2>/dev/null diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 06dc59cf5c..68ba1bc68e 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -49,9 +49,11 @@ FUZZ_TARGETS = \ test/fuzz/key \ test/fuzz/key_io \ test/fuzz/key_origin_info_deserialize \ + test/fuzz/kitchen_sink \ test/fuzz/locale \ test/fuzz/merkle_block_deserialize \ test/fuzz/merkleblock \ + test/fuzz/message \ test/fuzz/messageheader_deserialize \ test/fuzz/multiplication_overflow \ test/fuzz/net_permissions \ @@ -64,13 +66,13 @@ FUZZ_TARGETS = \ test/fuzz/parse_numbers \ test/fuzz/parse_script \ test/fuzz/parse_univalue \ - test/fuzz/prevector \ test/fuzz/partial_merkle_tree_deserialize \ test/fuzz/partially_signed_transaction_deserialize \ + test/fuzz/policy_estimator \ test/fuzz/pow \ test/fuzz/prefilled_transaction_deserialize \ + test/fuzz/prevector \ test/fuzz/primitives_transaction \ - test/fuzz/process_messages \ test/fuzz/process_message \ test/fuzz/process_message_addr \ test/fuzz/process_message_block \ @@ -96,12 +98,14 @@ FUZZ_TARGETS = \ test/fuzz/process_message_tx \ test/fuzz/process_message_verack \ test/fuzz/process_message_version \ + test/fuzz/process_messages \ test/fuzz/protocol \ test/fuzz/psbt \ test/fuzz/psbt_input_deserialize \ test/fuzz/psbt_output_deserialize \ test/fuzz/pub_key_deserialize \ test/fuzz/random \ + test/fuzz/rbf \ test/fuzz/rolling_bloom_filter \ test/fuzz/script \ test/fuzz/script_deserialize \ @@ -116,6 +120,7 @@ FUZZ_TARGETS = \ test/fuzz/string \ test/fuzz/strprintf \ test/fuzz/sub_net_deserialize \ + test/fuzz/system \ test/fuzz/timedata \ test/fuzz/transaction \ test/fuzz/tx_in \ @@ -567,6 +572,12 @@ test_fuzz_key_origin_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_key_origin_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_key_origin_info_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_kitchen_sink_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_kitchen_sink_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_kitchen_sink_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_kitchen_sink_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_kitchen_sink_SOURCES = test/fuzz/kitchen_sink.cpp + test_fuzz_locale_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_locale_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_locale_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -585,6 +596,12 @@ test_fuzz_merkleblock_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_merkleblock_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_merkleblock_SOURCES = test/fuzz/merkleblock.cpp +test_fuzz_message_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_message_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_message_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_message_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_message_SOURCES = test/fuzz/message.cpp + test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -675,6 +692,12 @@ test_fuzz_partially_signed_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMO test_fuzz_partially_signed_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_partially_signed_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_policy_estimator_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_policy_estimator_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_policy_estimator_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_policy_estimator_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_policy_estimator_SOURCES = test/fuzz/policy_estimator.cpp + test_fuzz_pow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_pow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_pow_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -885,6 +908,12 @@ test_fuzz_random_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_random_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_random_SOURCES = test/fuzz/random.cpp +test_fuzz_rbf_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_rbf_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_rbf_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_rbf_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_rbf_SOURCES = test/fuzz/rbf.cpp + test_fuzz_rolling_bloom_filter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_rolling_bloom_filter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_rolling_bloom_filter_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -969,6 +998,12 @@ test_fuzz_sub_net_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_sub_net_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_sub_net_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_system_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_system_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_system_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_system_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_system_SOURCES = test/fuzz/system.cpp + test_fuzz_timedata_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_timedata_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_timedata_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 1a0084c915..268f67cada 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -16,7 +16,14 @@ static void AssembleBlock(benchmark::State& state) { - RegTestingSetup test_setup; + TestingSetup test_setup{ + CBaseChainParams::REGTEST, + /* extra_args */ { + "-nodebuglogfile", + "-nodebug", + }, + }; + const std::vector<unsigned char> op_true{OP_TRUE}; CScriptWitness witness; witness.stack.push_back(op_true); diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index d658976c3c..86f9a0bf67 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -47,8 +47,6 @@ static void CCoinsCaching(benchmark::State& state) while (state.KeepRunning()) { bool success = AreInputsStandard(tx_1, coins); assert(success); - CAmount value = coins.GetValueIn(tx_1); - assert(value == (50 + 21 + 22) * COIN); } ECC_Stop(); } diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index 57673ccb84..e87f15042b 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -14,7 +14,13 @@ static void DuplicateInputs(benchmark::State& state) { - RegTestingSetup test_setup; + TestingSetup test_setup{ + CBaseChainParams::REGTEST, + /* extra_args */ { + "-nodebuglogfile", + "-nodebug", + }, + }; const CScript SCRIPT_PUB{CScript(OP_TRUE)}; diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 7df024def6..69483f2914 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -16,8 +16,8 @@ static void AddTx(const CTransactionRef& tx, const CAmount& nFee, CTxMemPool& po unsigned int sigOpCost = 4; LockPoints lp; pool.addUnchecked(CTxMemPoolEntry( - tx, nFee, nTime, nHeight, - spendsCoinbase, sigOpCost, lp)); + tx, nFee, nTime, nHeight, + spendsCoinbase, sigOpCost, lp)); } // Right now this is only testing eviction performance in an extremely small @@ -25,7 +25,13 @@ static void AddTx(const CTransactionRef& tx, const CAmount& nFee, CTxMemPool& po // unique transactions for a more meaningful performance measurement. static void MempoolEviction(benchmark::State& state) { - RegTestingSetup test_setup; + TestingSetup test_setup{ + CBaseChainParams::REGTEST, + /* extra_args */ { + "-nodebuglogfile", + "-nodebug", + }, + }; CMutableTransaction tx1 = CMutableTransaction(); tx1.vin.resize(1); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 05d61fca22..810c344ab5 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -14,7 +14,14 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const bool add_watchonly, const bool add_mine) { - RegTestingSetup test_setup; + TestingSetup test_setup{ + CBaseChainParams::REGTEST, + /* extra_args */ { + "-nodebuglogfile", + "-nodebug", + }, + }; + const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; NodeContext node; @@ -25,7 +32,7 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const b bool first_run; if (wallet.LoadWallet(first_run) != DBErrors::LOAD_OK) assert(false); } - auto handler = chain->handleNotifications({ &wallet, [](CWallet*) {} }); + auto handler = chain->handleNotifications({&wallet, [](CWallet*) {}}); const Optional<std::string> address_mine{add_mine ? Optional<std::string>{getnewaddress(wallet)} : nullopt}; if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); diff --git a/src/coins.cpp b/src/coins.cpp index 6b4cb2aec7..7b76c13f98 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -233,18 +233,6 @@ unsigned int CCoinsViewCache::GetCacheSize() const { return cacheCoins.size(); } -CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const -{ - if (tx.IsCoinBase()) - return 0; - - CAmount nResult = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - nResult += AccessCoin(tx.vin[i].prevout).out.nValue; - - return nResult; -} - bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const { if (!tx.IsCoinBase()) { diff --git a/src/coins.h b/src/coins.h index 2aed56c2bd..dcec741998 100644 --- a/src/coins.h +++ b/src/coins.h @@ -315,16 +315,6 @@ public: //! Calculate the size of the cache (in bytes) size_t DynamicMemoryUsage() const; - /** - * Amount of bitcoins coming in to a transaction - * Note that lightweight clients may not know anything besides the hash of previous transactions, - * so may not be able to calculate this. - * - * @param[in] tx transaction for which we are checking input total - * @return Sum of value of all inputs (scriptSigs) - */ - CAmount GetValueIn(const CTransaction& tx) const; - //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index a5582e3b2c..0f7848bae1 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -8,6 +8,7 @@ class CWallet; enum class WalletCreationStatus; +struct bilingual_str; namespace interfaces { class Chain; @@ -72,12 +73,12 @@ std::vector<std::shared_ptr<CWallet>> GetWallets() throw std::logic_error("Wallet function called in non-wallet build."); } -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::vector<std::string>& warnings) +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) { throw std::logic_error("Wallet function called in non-wallet build."); } -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result) +WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result) { throw std::logic_error("Wallet function called in non-wallet build."); } diff --git a/src/init.cpp b/src/init.cpp index a213dacbe0..3e2ee4d425 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -708,6 +708,10 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) break; // This error is logged in OpenBlockFile LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); LoadExternalBlockFile(chainparams, file, &pos); + if (ShutdownRequested()) { + LogPrintf("Shutdown requested. Exit %s\n", __func__); + return; + } nFile++; } pblocktree->WriteReindexing(false); @@ -723,6 +727,10 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) if (file) { LogPrintf("Importing blocks file %s...\n", path.string()); LoadExternalBlockFile(chainparams, file); + if (ShutdownRequested()) { + LogPrintf("Shutdown requested. Exit %s\n", __func__); + return; + } } else { LogPrintf("Warning: Could not open blocks file %s\n", path.string()); } diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index 0e7641ae32..c5262e4bc0 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -53,88 +53,6 @@ bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<Rec return true; } -class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> -{ - Optional<int> getHeight() override - { - LockAssertion lock(::cs_main); - int height = ::ChainActive().Height(); - if (height >= 0) { - return height; - } - return nullopt; - } - Optional<int> getBlockHeight(const uint256& hash) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = LookupBlockIndex(hash); - if (block && ::ChainActive().Contains(block)) { - return block->nHeight; - } - return nullopt; - } - uint256 getBlockHash(int height) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; - assert(block != nullptr); - return block->GetBlockHash(); - } - bool haveBlockOnDisk(int height) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; - return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; - } - Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive().FindEarliestAtLeast(time, height); - if (block) { - if (hash) *hash = block->GetBlockHash(); - return block->nHeight; - } - return nullopt; - } - Optional<int> findFork(const uint256& hash, Optional<int>* height) override - { - LockAssertion lock(::cs_main); - const CBlockIndex* block = LookupBlockIndex(hash); - const CBlockIndex* fork = block ? ::ChainActive().FindFork(block) : nullptr; - if (height) { - if (block) { - *height = block->nHeight; - } else { - height->reset(); - } - } - if (fork) { - return fork->nHeight; - } - return nullopt; - } - CBlockLocator getTipLocator() override - { - LockAssertion lock(::cs_main); - return ::ChainActive().GetLocator(); - } - Optional<int> findLocatorFork(const CBlockLocator& locator) override - { - LockAssertion lock(::cs_main); - if (CBlockIndex* fork = FindForkInGlobalIndex(::ChainActive(), locator)) { - return fork->nHeight; - } - return nullopt; - } - bool checkFinalTx(const CTransaction& tx) override - { - LockAssertion lock(::cs_main); - return CheckFinalTx(tx); - } - - using UniqueLock::UniqueLock; -}; - class NotificationsProxy : public CValidationInterface { public: @@ -227,12 +145,64 @@ class ChainImpl : public Chain { public: explicit ChainImpl(NodeContext& node) : m_node(node) {} - std::unique_ptr<Chain::Lock> lock(bool try_lock) override + Optional<int> getHeight() override + { + LOCK(::cs_main); + int height = ::ChainActive().Height(); + if (height >= 0) { + return height; + } + return nullopt; + } + Optional<int> getBlockHeight(const uint256& hash) override + { + LOCK(::cs_main); + CBlockIndex* block = LookupBlockIndex(hash); + if (block && ::ChainActive().Contains(block)) { + return block->nHeight; + } + return nullopt; + } + uint256 getBlockHash(int height) override + { + LOCK(::cs_main); + CBlockIndex* block = ::ChainActive()[height]; + assert(block); + return block->GetBlockHash(); + } + bool haveBlockOnDisk(int height) override + { + LOCK(cs_main); + CBlockIndex* block = ::ChainActive()[height]; + return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; + } + Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) override { - auto lock = MakeUnique<LockImpl>(::cs_main, "cs_main", __FILE__, __LINE__, try_lock); - if (try_lock && lock && !*lock) return {}; - std::unique_ptr<Chain::Lock> result = std::move(lock); // Temporary to avoid CWG 1579 - return result; + LOCK(cs_main); + CBlockIndex* block = ::ChainActive().FindEarliestAtLeast(time, height); + if (block) { + if (hash) *hash = block->GetBlockHash(); + return block->nHeight; + } + return nullopt; + } + CBlockLocator getTipLocator() override + { + LOCK(cs_main); + return ::ChainActive().GetLocator(); + } + bool checkFinalTx(const CTransaction& tx) override + { + LOCK(cs_main); + return CheckFinalTx(tx); + } + Optional<int> findLocatorFork(const CBlockLocator& locator) override + { + LOCK(cs_main); + if (CBlockIndex* fork = FindForkInGlobalIndex(::ChainActive(), locator)) { + return fork->nHeight; + } + return nullopt; } bool findBlock(const uint256& hash, const FoundBlock& block) override { diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index fb77c81cdc..e33fe54ac8 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -59,12 +59,7 @@ public: //! internal workings of the bitcoin node, and not being very convenient to use. //! Chain methods should be cleaned up and simplified over time. Examples: //! -//! * The Chain::lock() method, which lets clients delay chain tip updates -//! should be removed when clients are able to respond to updates -//! asynchronously -//! (https://github.com/bitcoin/bitcoin/pull/10973#issuecomment-380101269). -//! -//! * The initMessage() and showProgress() methods which the wallet uses to send +//! * The initMessages() and showProgress() methods which the wallet uses to send //! notifications to the GUI should go away when GUI and wallet can directly //! communicate with each other without going through the node //! (https://github.com/bitcoin/bitcoin/pull/15288#discussion_r253321096). @@ -72,68 +67,53 @@ public: //! * The handleRpc, registerRpcs, rpcEnableDeprecated methods and other RPC //! methods can go away if wallets listen for HTTP requests on their own //! ports instead of registering to handle requests on the node HTTP port. +//! +//! * Move fee estimation queries to an asynchronous interface and let the +//! wallet cache it, fee estimation being driven by node mempool, wallet +//! should be the consumer. +//! +//! * The `guessVerificationProgress`, `getBlockHeight`, `getBlockHash`, etc +//! methods can go away if rescan logic is moved on the node side, and wallet +//! only register rescan request. class Chain { public: virtual ~Chain() {} - //! Interface for querying locked chain state, used by legacy code that - //! assumes state won't change between calls. New code should avoid using - //! the Lock interface and instead call higher-level Chain methods - //! that return more information so the chain doesn't need to stay locked - //! between calls. - class Lock - { - public: - virtual ~Lock() {} - - //! Get current chain height, not including genesis block (returns 0 if - //! chain only contains genesis block, nullopt if chain does not contain - //! any blocks). - virtual Optional<int> getHeight() = 0; - - //! Get block height above genesis block. Returns 0 for genesis block, - //! 1 for following block, and so on. Returns nullopt for a block not - //! included in the current chain. - virtual Optional<int> getBlockHeight(const uint256& hash) = 0; - - //! Get block hash. Height must be valid or this function will abort. - virtual uint256 getBlockHash(int height) = 0; - - //! Check that the block is available on disk (i.e. has not been - //! pruned), and contains transactions. - virtual bool haveBlockOnDisk(int height) = 0; - - //! Return height of the first block in the chain with timestamp equal - //! or greater than the given time and height equal or greater than the - //! given height, or nullopt if there is no block with a high enough - //! timestamp and height. Also return the block hash as an optional output parameter - //! (to avoid the cost of a second lookup in case this information is needed.) - virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) = 0; - - //! Return height of the specified block if it is on the chain, otherwise - //! return the height of the highest block on chain that's an ancestor - //! of the specified block, or nullopt if there is no common ancestor. - //! Also return the height of the specified block as an optional output - //! parameter (to avoid the cost of a second hash lookup in case this - //! information is desired). - virtual Optional<int> findFork(const uint256& hash, Optional<int>* height) = 0; - - //! Get locator for the current chain tip. - virtual CBlockLocator getTipLocator() = 0; - - //! Return height of the highest block on chain in common with the locator, - //! which will either be the original block used to create the locator, - //! or one of its ancestors. - virtual Optional<int> findLocatorFork(const CBlockLocator& locator) = 0; - - //! Check if transaction will be final given chain height current time. - virtual bool checkFinalTx(const CTransaction& tx) = 0; - }; + //! Get current chain height, not including genesis block (returns 0 if + //! chain only contains genesis block, nullopt if chain does not contain + //! any blocks) + virtual Optional<int> getHeight() = 0; + + //! Get block height above genesis block. Returns 0 for genesis block, + //! 1 for following block, and so on. Returns nullopt for a block not + //! included in the current chain. + virtual Optional<int> getBlockHeight(const uint256& hash) = 0; + + //! Get block hash. Height must be valid or this function will abort. + virtual uint256 getBlockHash(int height) = 0; + + //! Check that the block is available on disk (i.e. has not been + //! pruned), and contains transactions. + virtual bool haveBlockOnDisk(int height) = 0; + + //! Return height of the first block in the chain with timestamp equal + //! or greater than the given time and height equal or greater than the + //! given height, or nullopt if there is no block with a high enough + //! timestamp and height. Also return the block hash as an optional output parameter + //! (to avoid the cost of a second lookup in case this information is needed.) + virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) = 0; + + //! Get locator for the current chain tip. + virtual CBlockLocator getTipLocator() = 0; + + //! Return height of the highest block on chain in common with the locator, + //! which will either be the original block used to create the locator, + //! or one of its ancestors. + virtual Optional<int> findLocatorFork(const CBlockLocator& locator) = 0; - //! Return Lock interface. Chain is locked when this is called, and - //! unlocked when the returned interface is freed. - virtual std::unique_ptr<Lock> lock(bool try_lock = false) = 0; + //! Check if transaction will be final given chain height current time. + virtual bool checkFinalTx(const CTransaction& tx) = 0; //! Return whether node has the block and optionally return block metadata //! or contents. diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 5ebbd61584..8564819e6a 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -43,8 +43,8 @@ class CWallet; fs::path GetWalletDir(); std::vector<fs::path> ListWalletDir(); std::vector<std::shared_ptr<CWallet>> GetWallets(); -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::vector<std::string>& warnings); -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result); +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings); +WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result); std::unique_ptr<interfaces::Handler> HandleLoadWallet(interfaces::Node::LoadWalletFn load_wallet); namespace interfaces { @@ -259,11 +259,11 @@ public: } return wallets; } - std::unique_ptr<Wallet> loadWallet(const std::string& name, std::string& error, std::vector<std::string>& warnings) override + std::unique_ptr<Wallet> loadWallet(const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) override { return MakeWallet(LoadWallet(*m_context.chain, name, error, warnings)); } - std::unique_ptr<Wallet> createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, WalletCreationStatus& status) override + std::unique_ptr<Wallet> createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, WalletCreationStatus& status) override { std::shared_ptr<CWallet> wallet; status = CreateWallet(*m_context.chain, passphrase, wallet_creation_flags, name, error, warnings, wallet); diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 53a20886cd..db9b42b293 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -27,9 +27,10 @@ class Coin; class RPCTimerInterface; class UniValue; class proxyType; +enum class WalletCreationStatus; struct CNodeStateStats; struct NodeContext; -enum class WalletCreationStatus; +struct bilingual_str; namespace interfaces { class Handler; @@ -201,10 +202,10 @@ public: //! Attempts to load a wallet from file or directory. //! The loaded wallet is also notified to handlers previously registered //! with handleLoadWallet. - virtual std::unique_ptr<Wallet> loadWallet(const std::string& name, std::string& error, std::vector<std::string>& warnings) = 0; + virtual std::unique_ptr<Wallet> loadWallet(const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) = 0; //! Create a wallet from file - virtual std::unique_ptr<Wallet> createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, WalletCreationStatus& status) = 0; + virtual std::unique_ptr<Wallet> createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, WalletCreationStatus& status) = 0; //! Register handler for init messages. using InitMessageFn = std::function<void(const std::string& message)>; diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 942952d9c6..13b034936b 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -60,7 +60,7 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) } //! Construct wallet tx status struct. -WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx) +WalletTxStatus MakeWalletTxStatus(CWallet& wallet, const CWalletTx& wtx) { WalletTxStatus result; result.block_height = wtx.m_confirm.block_height > 0 ? wtx.m_confirm.block_height : std::numeric_limits<int>::max(); @@ -68,8 +68,8 @@ WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const C result.depth_in_main_chain = wtx.GetDepthInMainChain(); result.time_received = wtx.nTimeReceived; result.lock_time = wtx.tx->nLockTime; - result.is_final = locked_chain.checkFinalTx(*wtx.tx); - result.is_trusted = wtx.IsTrusted(locked_chain); + result.is_final = wallet.chain().checkFinalTx(*wtx.tx); + result.is_trusted = wtx.IsTrusted(); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); result.is_in_main_chain = wtx.IsInMainChain(); @@ -196,25 +196,21 @@ public: } void lockCoin(const COutPoint& output) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->LockCoin(output); } void unlockCoin(const COutPoint& output) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->UnlockCoin(output); } bool isLockedCoin(const COutPoint& output) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->IsLockedCoin(output.hash, output.n); } void listLockedCoins(std::vector<COutPoint>& outputs) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->ListLockedCoins(outputs); } @@ -223,12 +219,11 @@ public: bool sign, int& change_pos, CAmount& fee, - std::string& fail_reason) override + bilingual_str& fail_reason) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); CTransactionRef tx; - if (!m_wallet->CreateTransaction(*locked_chain, recipients, tx, fee, change_pos, + if (!m_wallet->CreateTransaction(recipients, tx, fee, change_pos, fail_reason, coin_control, sign)) { return {}; } @@ -238,14 +233,12 @@ public: WalletValueMap value_map, WalletOrderForm order_form) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); m_wallet->CommitTransaction(std::move(tx), std::move(value_map), std::move(order_form)); } bool transactionCanBeAbandoned(const uint256& txid) override { return m_wallet->TransactionCanBeAbandoned(txid); } bool abandonTransaction(const uint256& txid) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->AbandonTransaction(txid); } @@ -255,7 +248,7 @@ public: } bool createBumpTransaction(const uint256& txid, const CCoinControl& coin_control, - std::vector<std::string>& errors, + std::vector<bilingual_str>& errors, CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) override @@ -265,7 +258,7 @@ public: bool signBumpTransaction(CMutableTransaction& mtx) override { return feebumper::SignTransaction(*m_wallet.get(), mtx); } bool commitBumpTransaction(const uint256& txid, CMutableTransaction&& mtx, - std::vector<std::string>& errors, + std::vector<bilingual_str>& errors, uint256& bumped_txid) override { return feebumper::CommitTransaction(*m_wallet.get(), txid, std::move(mtx), errors, bumped_txid) == @@ -273,7 +266,6 @@ public: } CTransactionRef getTx(const uint256& txid) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); auto mi = m_wallet->mapWallet.find(txid); if (mi != m_wallet->mapWallet.end()) { @@ -283,7 +275,6 @@ public: } WalletTx getWalletTx(const uint256& txid) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); auto mi = m_wallet->mapWallet.find(txid); if (mi != m_wallet->mapWallet.end()) { @@ -293,7 +284,6 @@ public: } std::vector<WalletTx> getWalletTxs() override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); std::vector<WalletTx> result; result.reserve(m_wallet->mapWallet.size()); @@ -307,10 +297,6 @@ public: int& num_blocks, int64_t& block_time) override { - auto locked_chain = m_wallet->chain().lock(true /* try_lock */); - if (!locked_chain) { - return false; - } TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; @@ -322,7 +308,7 @@ public: num_blocks = m_wallet->GetLastBlockHeight(); block_time = -1; CHECK_NONFATAL(m_wallet->chain().findBlock(m_wallet->GetLastBlockHash(), FoundBlock().time(block_time))); - tx_status = MakeWalletTxStatus(*locked_chain, mi->second); + tx_status = MakeWalletTxStatus(*m_wallet, mi->second); return true; } WalletTx getWalletTxDetails(const uint256& txid, @@ -331,14 +317,13 @@ public: bool& in_mempool, int& num_blocks) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); auto mi = m_wallet->mapWallet.find(txid); if (mi != m_wallet->mapWallet.end()) { - num_blocks = locked_chain->getHeight().get_value_or(-1); + num_blocks = m_wallet->GetLastBlockHeight(); in_mempool = mi->second.InMempool(); order_form = mi->second.vOrderForm; - tx_status = MakeWalletTxStatus(*locked_chain, mi->second); + tx_status = MakeWalletTxStatus(*m_wallet, mi->second); return MakeWalletTx(*m_wallet, mi->second); } return {}; @@ -368,8 +353,6 @@ public: } bool tryGetBalances(WalletBalances& balances, int& num_blocks, bool force, int cached_num_blocks) override { - auto locked_chain = m_wallet->chain().lock(true /* try_lock */); - if (!locked_chain) return false; TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; @@ -386,34 +369,29 @@ public: } isminetype txinIsMine(const CTxIn& txin) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->IsMine(txin); } isminetype txoutIsMine(const CTxOut& txout) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->IsMine(txout); } CAmount getDebit(const CTxIn& txin, isminefilter filter) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->GetDebit(txin, filter); } CAmount getCredit(const CTxOut& txout, isminefilter filter) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->GetCredit(txout, filter); } CoinsList listCoins() override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); CoinsList result; - for (const auto& entry : m_wallet->ListCoins(*locked_chain)) { + for (const auto& entry : m_wallet->ListCoins()) { auto& group = result[entry.first]; for (const auto& coin : entry.second) { group.emplace_back(COutPoint(coin.tx->GetHash(), coin.i), @@ -424,7 +402,6 @@ public: } std::vector<WalletTxOut> getCoins(const std::vector<COutPoint>& outputs) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); std::vector<WalletTxOut> result; result.reserve(outputs.size()); @@ -464,6 +441,7 @@ public: { RemoveWallet(m_wallet); } + bool isLegacy() override { return m_wallet->IsLegacy(); } std::unique_ptr<Handler> handleUnload(UnloadFn fn) override { return MakeHandler(m_wallet->NotifyUnload.connect(fn)); @@ -495,6 +473,7 @@ public: { return MakeHandler(m_wallet->NotifyCanGetAddressesChanged.connect(fn)); } + CWallet* wallet() override { return m_wallet.get(); } std::shared_ptr<CWallet> m_wallet; }; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 0157e9cf95..f35335c69f 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -31,6 +31,7 @@ enum class TransactionError; enum isminetype : unsigned int; struct CRecipient; struct PartiallySignedTransaction; +struct bilingual_str; typedef uint8_t isminefilter; namespace interfaces { @@ -136,7 +137,7 @@ public: bool sign, int& change_pos, CAmount& fee, - std::string& fail_reason) = 0; + bilingual_str& fail_reason) = 0; //! Commit transaction. virtual void commitTransaction(CTransactionRef tx, @@ -155,7 +156,7 @@ public: //! Create bump transaction. virtual bool createBumpTransaction(const uint256& txid, const CCoinControl& coin_control, - std::vector<std::string>& errors, + std::vector<bilingual_str>& errors, CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) = 0; @@ -166,7 +167,7 @@ public: //! Commit bump transaction. virtual bool commitBumpTransaction(const uint256& txid, CMutableTransaction&& mtx, - std::vector<std::string>& errors, + std::vector<bilingual_str>& errors, uint256& bumped_txid) = 0; //! Get a transaction. @@ -266,6 +267,9 @@ public: // Remove wallet. virtual void remove() = 0; + //! Return whether is a legacy wallet + virtual bool isLegacy() = 0; + //! Register handler for unload message. using UnloadFn = std::function<void()>; virtual std::unique_ptr<Handler> handleUnload(UnloadFn fn) = 0; @@ -297,6 +301,9 @@ public: //! Register handler for keypool changed messages. using CanGetAddressesChangedFn = std::function<void()>; virtual std::unique_ptr<Handler> handleCanGetAddressesChanged(CanGetAddressesChangedFn fn) = 0; + + //! Return pointer to internal wallet class, useful for testing. + virtual CWallet* wallet() { return nullptr; } }; //! Information about one wallet address. diff --git a/src/logging/timer.h b/src/logging/timer.h index 21bb3d121e..159920e397 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -93,12 +93,10 @@ private: } // namespace BCLog -#define LOG_TIME_MICROS(end_msg, ...) \ - BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) -#define LOG_TIME_MILLIS(end_msg, ...) \ - BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) -#define LOG_TIME_SECONDS(end_msg, ...) \ - BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) +#define LOG_TIME_MILLIS_WITH_CATEGORY(end_msg, log_category) \ + BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category) +#define LOG_TIME_SECONDS(end_msg) \ + BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg) #endif // BITCOIN_LOGGING_TIMER_H diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 26327ac6eb..50a8a8a882 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -810,6 +810,19 @@ void PeerLogicValidation::InitializeNode(CNode *pnode) { PushNodeVersion(pnode, connman, GetTime()); } +void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const +{ + std::set<uint256> unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); + + for (const uint256& txid : unbroadcast_txids) { + RelayTransaction(txid, *connman); + } + + // schedule next run for 10-15 minutes in the future + const std::chrono::milliseconds delta = std::chrono::minutes{10} + GetRandMillis(std::chrono::minutes{5}); + scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); +} + void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { fUpdateConnectionTime = false; LOCK(cs_main); @@ -1159,6 +1172,10 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CS // timer. static_assert(EXTRA_PEER_CHECK_INTERVAL < STALE_CHECK_INTERVAL, "peer eviction timer should be less than stale tip check timer"); scheduler.scheduleEvery([this, consensusParams] { this->CheckForStaleTipAndEvictPeers(consensusParams); }, std::chrono::seconds{EXTRA_PEER_CHECK_INTERVAL}); + + // schedule next run for 10-15 minutes in the future + const std::chrono::milliseconds delta = std::chrono::minutes{10} + GetRandMillis(std::chrono::minutes{5}); + scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); } /** @@ -1587,7 +1604,7 @@ void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, c } } -void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnman* connman, const CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) +void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnman* connman, CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) { AssertLockNotHeld(cs_main); @@ -1636,7 +1653,13 @@ void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnm push = true; } } - if (!push) { + + if (push) { + // We interpret fulfilling a GETDATA for a transaction as a + // successful initial broadcast and remove it from our + // unbroadcast set. + mempool.RemoveUnbroadcastTx(inv.hash); + } else { vNotFound.push_back(inv); } } diff --git a/src/net_processing.h b/src/net_processing.h index 3d9bc65a3a..a85d5e7c70 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -76,6 +76,8 @@ public: void CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams); /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ void EvictExtraOutboundPeers(int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Retrieve unbroadcast transactions from the mempool and reattempt sending to peers */ + void ReattemptInitialBroadcast(CScheduler& scheduler) const; private: int64_t m_stale_tip_check_time; //!< Next time to check for stale tip diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 201406ce3b..3841d8687d 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -78,6 +78,10 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t } if (relay) { + // the mempool tracks locally submitted transactions to make a + // best-effort of initial broadcast + node.mempool->AddUnbroadcastTx(hashTx); + RelayTransaction(hashTx, *node.connman); } diff --git a/src/prevector.h b/src/prevector.h index 9f2f7ba2de..aa20efaaa7 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -152,7 +152,7 @@ private: struct { char* indirect; size_type capacity; - }; + } indirect_contents; }; #pragma pack(pop) alignas(char*) direct_or_indirect _union = {}; @@ -163,8 +163,8 @@ private: T* direct_ptr(difference_type pos) { return reinterpret_cast<T*>(_union.direct) + pos; } const T* direct_ptr(difference_type pos) const { return reinterpret_cast<const T*>(_union.direct) + pos; } - T* indirect_ptr(difference_type pos) { return reinterpret_cast<T*>(_union.indirect) + pos; } - const T* indirect_ptr(difference_type pos) const { return reinterpret_cast<const T*>(_union.indirect) + pos; } + T* indirect_ptr(difference_type pos) { return reinterpret_cast<T*>(_union.indirect_contents.indirect) + pos; } + const T* indirect_ptr(difference_type pos) const { return reinterpret_cast<const T*>(_union.indirect_contents.indirect) + pos; } bool is_direct() const { return _size <= N; } void change_capacity(size_type new_capacity) { @@ -182,17 +182,17 @@ private: /* FIXME: Because malloc/realloc here won't call new_handler if allocation fails, assert success. These should instead use an allocator or new/delete so that handlers are called as necessary, but performance would be slightly degraded by doing so. */ - _union.indirect = static_cast<char*>(realloc(_union.indirect, ((size_t)sizeof(T)) * new_capacity)); - assert(_union.indirect); - _union.capacity = new_capacity; + _union.indirect_contents.indirect = static_cast<char*>(realloc(_union.indirect_contents.indirect, ((size_t)sizeof(T)) * new_capacity)); + assert(_union.indirect_contents.indirect); + _union.indirect_contents.capacity = new_capacity; } else { char* new_indirect = static_cast<char*>(malloc(((size_t)sizeof(T)) * new_capacity)); assert(new_indirect); T* src = direct_ptr(0); T* dst = reinterpret_cast<T*>(new_indirect); memcpy(dst, src, size() * sizeof(T)); - _union.indirect = new_indirect; - _union.capacity = new_capacity; + _union.indirect_contents.indirect = new_indirect; + _union.indirect_contents.capacity = new_capacity; _size += N + 1; } } @@ -301,7 +301,7 @@ public: if (is_direct()) { return N; } else { - return _union.capacity; + return _union.indirect_contents.capacity; } } @@ -468,8 +468,8 @@ public: clear(); } if (!is_direct()) { - free(_union.indirect); - _union.indirect = nullptr; + free(_union.indirect_contents.indirect); + _union.indirect_contents.indirect = nullptr; } } @@ -521,7 +521,7 @@ public: if (is_direct()) { return 0; } else { - return ((size_t)(sizeof(T))) * _union.capacity; + return ((size_t)(sizeof(T))) * _union.indirect_contents.capacity; } } diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 00ccbc32f9..58b3e8aedc 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -324,8 +324,6 @@ public: // Return sum of txouts. CAmount GetValueOut() const; - // GetValueIn() is a method on CCoinsViewCache, because - // inputs must be known to compute value in. /** * Get the total transaction size in bytes, including witness data. diff --git a/src/pubkey.h b/src/pubkey.h index 2fc92c9bc6..261842b7f7 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -219,6 +219,11 @@ struct CExtPubKey { a.pubkey == b.pubkey; } + friend bool operator!=(const CExtPubKey &a, const CExtPubKey &b) + { + return !(a == b); + } + void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const; void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); bool Derive(CExtPubKey& out, unsigned int nChild) const; diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index 1aaf33c6a4..f48f28d03a 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -136,6 +136,8 @@ AddressBookPage::AddressBookPage(const PlatformStyle *platformStyle, Mode _mode, connect(ui->tableView, &QWidget::customContextMenuRequested, this, &AddressBookPage::contextualMenu); connect(ui->closeButton, &QPushButton::clicked, this, &QDialog::accept); + + GUIUtil::handleCloseWindowShortcut(this); } AddressBookPage::~AddressBookPage() diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index 67e7704551..3d1963b6e6 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -10,6 +10,7 @@ #include <qt/forms/ui_askpassphrasedialog.h> #include <qt/guiconstants.h> +#include <qt/guiutil.h> #include <qt/walletmodel.h> #include <support/allocators/secure.h> @@ -75,6 +76,8 @@ AskPassphraseDialog::AskPassphraseDialog(Mode _mode, QWidget *parent, SecureStri connect(ui->passEdit1, &QLineEdit::textChanged, this, &AskPassphraseDialog::textChanged); connect(ui->passEdit2, &QLineEdit::textChanged, this, &AskPassphraseDialog::textChanged); connect(ui->passEdit3, &QLineEdit::textChanged, this, &AskPassphraseDialog::textChanged); + + GUIUtil::handleCloseWindowShortcut(this); } AskPassphraseDialog::~AskPassphraseDialog() diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 71e4d59ea7..cf2de1a417 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -211,6 +211,8 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty #ifdef Q_OS_MAC m_app_nap_inhibitor = new CAppNapInhibitor; #endif + + GUIUtil::handleCloseWindowShortcut(this); } BitcoinGUI::~BitcoinGUI() diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 9495ba389a..b46f416b6c 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -134,6 +134,8 @@ CoinControlDialog::CoinControlDialog(const PlatformStyle *_platformStyle, QWidge ui->radioTreeMode->click(); if (settings.contains("nCoinControlSortColumn") && settings.contains("nCoinControlSortOrder")) sortView(settings.value("nCoinControlSortColumn").toInt(), (static_cast<Qt::SortOrder>(settings.value("nCoinControlSortOrder").toInt()))); + + GUIUtil::handleCloseWindowShortcut(this); } CoinControlDialog::~CoinControlDialog() @@ -379,14 +381,6 @@ void CoinControlDialog::viewItemChanged(QTreeWidgetItem* item, int column) if (ui->treeWidget->isEnabled()) // do not update on every click for (un)select all CoinControlDialog::updateLabels(model, this); } - - // TODO: Remove this temporary qt5 fix after Qt5.3 and Qt5.4 are no longer used. - // Fixed in Qt5.5 and above: https://bugreports.qt.io/browse/QTBUG-43473 - else if (column == COLUMN_CHECKBOX && item->childCount() > 0) - { - if (item->checkState(COLUMN_CHECKBOX) == Qt::PartiallyChecked && item->child(0)->checkState(COLUMN_CHECKBOX) == Qt::PartiallyChecked) - item->setCheckState(COLUMN_CHECKBOX, Qt::Checked); - } } // shows count of locked unspent outputs diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index 8e6474b0d4..5056e487fc 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -60,3 +60,8 @@ bool CreateWalletDialog::isMakeBlankWalletChecked() const { return ui->blank_wallet_checkbox->isChecked(); } + +bool CreateWalletDialog::isDescriptorWalletChecked() const +{ + return ui->descriptor_checkbox->isChecked(); +} diff --git a/src/qt/createwalletdialog.h b/src/qt/createwalletdialog.h index 30766107b9..20cce937c8 100644 --- a/src/qt/createwalletdialog.h +++ b/src/qt/createwalletdialog.h @@ -27,6 +27,7 @@ public: bool isEncryptWalletChecked() const; bool isDisablePrivateKeysChecked() const; bool isMakeBlankWalletChecked() const; + bool isDescriptorWalletChecked() const; private: Ui::CreateWalletDialog *ui; diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index 4711412ac4..e0af9a20a8 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -43,6 +43,8 @@ EditAddressDialog::EditAddressDialog(Mode _mode, QWidget *parent) : GUIUtil::ItemDelegate* delegate = new GUIUtil::ItemDelegate(mapper); connect(delegate, &GUIUtil::ItemDelegate::keyEscapePressed, this, &EditAddressDialog::reject); mapper->setItemDelegate(delegate); + + GUIUtil::handleCloseWindowShortcut(this); } EditAddressDialog::~EditAddressDialog() diff --git a/src/qt/forms/createwalletdialog.ui b/src/qt/forms/createwalletdialog.ui index e49bab8f3b..b592140dd7 100644 --- a/src/qt/forms/createwalletdialog.ui +++ b/src/qt/forms/createwalletdialog.ui @@ -7,7 +7,7 @@ <x>0</x> <y>0</y> <width>364</width> - <height>185</height> + <height>213</height> </rect> </property> <property name="windowTitle"> @@ -17,7 +17,7 @@ <property name="geometry"> <rect> <x>10</x> - <y>140</y> + <y>170</y> <width>341</width> <height>32</height> </rect> @@ -106,6 +106,22 @@ <string>Make Blank Wallet</string> </property> </widget> + <widget class="QCheckBox" name="descriptor_checkbox"> + <property name="geometry"> + <rect> + <x>20</x> + <y>140</y> + <width>171</width> + <height>22</height> + </rect> + </property> + <property name="toolTip"> + <string>Use descriptors for scriptPubKey management</string> + </property> + <property name="text"> + <string>Descriptor Wallet</string> + </property> + </widget> </widget> <tabstops> <tabstop>wallet_name_line_edit</tabstop> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index a54b65961c..af86fe5d27 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -54,6 +54,7 @@ #include <QSettings> #include <QSize> #include <QString> +#include <QShortcut> #include <QTextDocument> // for Qt::mightBeRichText #include <QThread> #include <QUrlQuery> @@ -378,6 +379,11 @@ void bringToFront(QWidget* w) } } +void handleCloseWindowShortcut(QWidget* w) +{ + QObject::connect(new QShortcut(QKeySequence(Qt::CTRL + Qt::Key_W), w), &QShortcut::activated, w, &QWidget::close); +} + void openDebugLogfile() { fs::path pathDebug = GetDataDir() / "debug.log"; diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index e571262443..0d9293fbad 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -124,6 +124,9 @@ namespace GUIUtil // Activate, show and raise the widget void bringToFront(QWidget* w); + // Set shortcut to close window + void handleCloseWindowShortcut(QWidget* w); + // Open debug.log void openDebugLogfile(); diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index b9dea2f8bf..9a3d43c2a6 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -15,6 +15,8 @@ OpenURIDialog::OpenURIDialog(QWidget *parent) : ui(new Ui::OpenURIDialog) { ui->setupUi(this); + + GUIUtil::handleCloseWindowShortcut(this); } OpenURIDialog::~OpenURIDialog() diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 8ee6c947e6..ae6aeb7709 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -135,6 +135,8 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : ui->minimizeToTray->setChecked(false); ui->minimizeToTray->setEnabled(false); } + + GUIUtil::handleCloseWindowShortcut(this); } OptionsDialog::~OptionsDialog() diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index c376921b72..e0ae1f9e92 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -161,20 +161,27 @@ void OverviewPage::setBalance(const interfaces::WalletBalances& balances) { int unit = walletModel->getOptionsModel()->getDisplayUnit(); m_balances = balances; - if (walletModel->wallet().privateKeysDisabled()) { - ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + if (walletModel->wallet().isLegacy()) { + if (walletModel->wallet().privateKeysDisabled()) { + ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + } else { + ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.balance, false, BitcoinUnits::separatorAlways)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_balance, false, BitcoinUnits::separatorAlways)); + ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_balance, false, BitcoinUnits::separatorAlways)); + ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchAvailable->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchPending->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + } } else { ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.balance, false, BitcoinUnits::separatorAlways)); ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_balance, false, BitcoinUnits::separatorAlways)); ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_balance, false, BitcoinUnits::separatorAlways)); ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchAvailable->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchPending->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); } // only show immature (newly mined) balance if it's non-zero, so as not to complicate things // for the non-mining users diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 180550c5ae..5debded4ea 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -157,17 +157,40 @@ void ReceiveCoinsDialog::on_receiveButton_clicked() } } address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, "", address_type); - SendCoinsRecipient info(address, label, - ui->reqAmount->value(), ui->reqMessage->text()); - ReceiveRequestDialog *dialog = new ReceiveRequestDialog(this); - dialog->setAttribute(Qt::WA_DeleteOnClose); - dialog->setModel(model); - dialog->setInfo(info); - dialog->show(); - clear(); - /* Store request for later reference */ - model->getRecentRequestsTableModel()->addNewRequest(info); + switch(model->getAddressTableModel()->getEditStatus()) + { + case AddressTableModel::EditStatus::OK: { + // Success + SendCoinsRecipient info(address, label, + ui->reqAmount->value(), ui->reqMessage->text()); + ReceiveRequestDialog *dialog = new ReceiveRequestDialog(this); + dialog->setAttribute(Qt::WA_DeleteOnClose); + dialog->setModel(model); + dialog->setInfo(info); + dialog->show(); + + /* Store request for later reference */ + model->getRecentRequestsTableModel()->addNewRequest(info); + break; + } + case AddressTableModel::EditStatus::WALLET_UNLOCK_FAILURE: + QMessageBox::critical(this, windowTitle(), + tr("Could not unlock wallet."), + QMessageBox::Ok, QMessageBox::Ok); + break; + case AddressTableModel::EditStatus::KEY_GENERATION_FAILURE: + QMessageBox::critical(this, windowTitle(), + tr("Could not generate new %1 address").arg(QString::fromStdString(FormatOutputType(address_type))), + QMessageBox::Ok, QMessageBox::Ok); + break; + // These aren't valid return values for our action + case AddressTableModel::EditStatus::INVALID_ADDRESS: + case AddressTableModel::EditStatus::DUPLICATE_ADDRESS: + case AddressTableModel::EditStatus::NO_CHANGES: + assert(false); + } + clear(); } void ReceiveCoinsDialog::on_recentRequestsView_doubleClicked(const QModelIndex &index) diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index b4fae7d78d..30bd5c6a5a 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -30,6 +30,8 @@ ReceiveRequestDialog::ReceiveRequestDialog(QWidget *parent) : #endif connect(ui->btnSaveAs, &QPushButton::clicked, ui->lblQRCode, &QRImageWidget::saveImage); + + GUIUtil::handleCloseWindowShortcut(this); } ReceiveRequestDialog::~ReceiveRequestDialog() diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 18a7e9b021..bfe316bdcd 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -498,6 +498,8 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty consoleFontSize = settings.value(fontSizeSettingsKey, QFontInfo(QFont()).pointSize()).toInt(); clear(); + + GUIUtil::handleCloseWindowShortcut(this); } RPCConsole::~RPCConsole() diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 67c2c71e9c..5cfbb67449 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -47,6 +47,8 @@ SignVerifyMessageDialog::SignVerifyMessageDialog(const PlatformStyle *_platformS ui->signatureOut_SM->setFont(GUIUtil::fixedPitchFont()); ui->signatureIn_VM->setFont(GUIUtil::fixedPitchFont()); + + GUIUtil::handleCloseWindowShortcut(this); } SignVerifyMessageDialog::~SignVerifyMessageDialog() diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index d8fe4a9238..ced6a299d5 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -127,6 +127,8 @@ SplashScreen::SplashScreen(interfaces::Node& node, Qt::WindowFlags f, const Netw subscribeToCoreSignals(); installEventFilter(this); + + GUIUtil::handleCloseWindowShortcut(this); } SplashScreen::~SplashScreen() diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 94a1e0a63d..2ee9ae0d86 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -145,7 +145,6 @@ void TestGUI(interfaces::Node& node) wallet->LoadWallet(firstRun); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); - auto locked_chain = wallet->chain().lock(); LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); wallet->SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type), "", "receive"); spk_man->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); diff --git a/src/qt/transactiondescdialog.cpp b/src/qt/transactiondescdialog.cpp index ca72720e00..715e312b19 100644 --- a/src/qt/transactiondescdialog.cpp +++ b/src/qt/transactiondescdialog.cpp @@ -5,6 +5,7 @@ #include <qt/transactiondescdialog.h> #include <qt/forms/ui_transactiondescdialog.h> +#include <qt/guiutil.h> #include <qt/transactiontablemodel.h> #include <QModelIndex> @@ -17,6 +18,8 @@ TransactionDescDialog::TransactionDescDialog(const QModelIndex &idx, QWidget *pa setWindowTitle(tr("Details for %1").arg(idx.data(TransactionTableModel::TxHashRole).toString())); QString desc = idx.data(TransactionTableModel::LongDescriptionRole).toString(); ui->detailText->setHtml(desc); + + GUIUtil::handleCloseWindowShortcut(this); } TransactionDescDialog::~TransactionDescDialog() diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index d16feba1bb..01922cf996 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -10,6 +10,8 @@ #include <qt/forms/ui_helpmessagedialog.h> +#include <qt/guiutil.h> + #include <clientversion.h> #include <init.h> #include <util/system.h> @@ -102,6 +104,8 @@ HelpMessageDialog::HelpMessageDialog(interfaces::Node& node, QWidget *parent, bo ui->scrollArea->setVisible(false); ui->aboutLogo->setVisible(false); } + + GUIUtil::handleCloseWindowShortcut(this); } HelpMessageDialog::~HelpMessageDialog() @@ -141,6 +145,8 @@ ShutdownWindow::ShutdownWindow(QWidget *parent, Qt::WindowFlags f): tr("%1 is shutting down...").arg(PACKAGE_NAME) + "<br /><br />" + tr("Do not shut down the computer until this window disappears."))); setLayout(layout); + + GUIUtil::handleCloseWindowShortcut(this); } QWidget* ShutdownWindow::showShutdownWindow(QMainWindow* window) diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 8e422fa962..7cde3ca30b 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -14,6 +14,7 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <util/string.h> +#include <util/translation.h> #include <wallet/wallet.h> #include <algorithm> @@ -226,6 +227,9 @@ void CreateWalletActivity::createWallet() if (m_create_wallet_dialog->isMakeBlankWalletChecked()) { flags |= WALLET_FLAG_BLANK_WALLET; } + if (m_create_wallet_dialog->isDescriptorWalletChecked()) { + flags |= WALLET_FLAG_DESCRIPTORS; + } QTimer::singleShot(500, worker(), [this, name, flags] { WalletCreationStatus status; @@ -241,10 +245,10 @@ void CreateWalletActivity::finish() { destroyProgressDialog(); - if (!m_error_message.empty()) { - QMessageBox::critical(m_parent_widget, tr("Create wallet failed"), QString::fromStdString(m_error_message)); + if (!m_error_message.original.empty()) { + QMessageBox::critical(m_parent_widget, tr("Create wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { - QMessageBox::warning(m_parent_widget, tr("Create wallet warning"), QString::fromStdString(Join(m_warning_message, "\n"))); + QMessageBox::warning(m_parent_widget, tr("Create wallet warning"), QString::fromStdString(Join(m_warning_message, "\n", OpTranslated))); } if (m_wallet_model) Q_EMIT created(m_wallet_model); @@ -282,10 +286,10 @@ void OpenWalletActivity::finish() { destroyProgressDialog(); - if (!m_error_message.empty()) { - QMessageBox::critical(m_parent_widget, tr("Open wallet failed"), QString::fromStdString(m_error_message)); + if (!m_error_message.original.empty()) { + QMessageBox::critical(m_parent_widget, tr("Open wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { - QMessageBox::warning(m_parent_widget, tr("Open wallet warning"), QString::fromStdString(Join(m_warning_message, "\n"))); + QMessageBox::warning(m_parent_widget, tr("Open wallet warning"), QString::fromStdString(Join(m_warning_message, "\n", OpTranslated))); } if (m_wallet_model) Q_EMIT opened(m_wallet_model); diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index 3808b7d8bf..24dd83adf7 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -8,6 +8,7 @@ #include <qt/sendcoinsrecipient.h> #include <support/allocators/secure.h> #include <sync.h> +#include <util/translation.h> #include <map> #include <memory> @@ -104,8 +105,8 @@ protected: QWidget* const m_parent_widget; QProgressDialog* m_progress_dialog{nullptr}; WalletModel* m_wallet_model{nullptr}; - std::string m_error_message; - std::vector<std::string> m_warning_message; + bilingual_str m_error_message; + std::vector<bilingual_str> m_warning_message; }; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index e7581cd64e..70ee7f4917 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -24,6 +24,7 @@ #include <psbt.h> #include <ui_interface.h> #include <util/system.h> // for GetBoolArg +#include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/wallet.h> // for CRecipient @@ -185,10 +186,10 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact { CAmount nFeeRequired = 0; int nChangePosRet = -1; - std::string strFailReason; + bilingual_str error; auto& newTx = transaction.getWtx(); - newTx = m_wallet->createTransaction(vecSend, coinControl, !wallet().privateKeysDisabled() /* sign */, nChangePosRet, nFeeRequired, strFailReason); + newTx = m_wallet->createTransaction(vecSend, coinControl, !wallet().privateKeysDisabled() /* sign */, nChangePosRet, nFeeRequired, error); transaction.setTransactionFee(nFeeRequired); if (fSubtractFeeFromAmount && newTx) transaction.reassignAmounts(nChangePosRet); @@ -199,8 +200,8 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact { return SendCoinsReturn(AmountWithFeeExceedsBalance); } - Q_EMIT message(tr("Send Coins"), QString::fromStdString(strFailReason), - CClientUIInterface::MSG_ERROR); + Q_EMIT message(tr("Send Coins"), QString::fromStdString(error.translated), + CClientUIInterface::MSG_ERROR); return TransactionCreationFailed; } @@ -482,14 +483,14 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) { CCoinControl coin_control; coin_control.m_signal_bip125_rbf = true; - std::vector<std::string> errors; + std::vector<bilingual_str> errors; CAmount old_fee; CAmount new_fee; CMutableTransaction mtx; if (!m_wallet->createBumpTransaction(hash, coin_control, errors, old_fee, new_fee, mtx)) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Increasing transaction fee failed") + "<br />(" + - (errors.size() ? QString::fromStdString(errors[0]) : "") +")"); - return false; + (errors.size() ? QString::fromStdString(errors[0].translated) : "") +")"); + return false; } const bool create_psbt = m_wallet->privateKeysDisabled(); @@ -551,8 +552,8 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) // commit the bumped transaction if(!m_wallet->commitBumpTransaction(hash, std::move(mtx), errors, new_hash)) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Could not commit transaction") + "<br />(" + - QString::fromStdString(errors[0])+")"); - return false; + QString::fromStdString(errors[0].translated)+")"); + return false; } return true; } diff --git a/src/random.cpp b/src/random.cpp index 765239e1f5..b408b1e13e 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -592,6 +592,11 @@ std::chrono::microseconds GetRandMicros(std::chrono::microseconds duration_max) return std::chrono::microseconds{GetRand(duration_max.count())}; } +std::chrono::milliseconds GetRandMillis(std::chrono::milliseconds duration_max) noexcept +{ + return std::chrono::milliseconds{GetRand(duration_max.count())}; +} + int GetRandInt(int nMax) noexcept { return GetRand(nMax); diff --git a/src/random.h b/src/random.h index 0c5008685b..690125079b 100644 --- a/src/random.h +++ b/src/random.h @@ -69,6 +69,7 @@ void GetRandBytes(unsigned char* buf, int num) noexcept; uint64_t GetRand(uint64_t nMax) noexcept; std::chrono::microseconds GetRandMicros(std::chrono::microseconds duration_max) noexcept; +std::chrono::milliseconds GetRandMillis(std::chrono::milliseconds duration_max) noexcept; int GetRandInt(int nMax) noexcept; uint256 GetRandHash() noexcept; diff --git a/src/rest.cpp b/src/rest.cpp index 0629557584..5f99e26bad 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -206,7 +206,7 @@ static bool rest_headers(HTTPRequest* req, return true; } default: { - return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex, .json)"); } } } @@ -607,7 +607,7 @@ static bool rest_blockhash_by_height(HTTPRequest* req, std::string height_str; const RetFormat rf = ParseDataFormat(height_str, str_uri_part); - int32_t blockheight; + int32_t blockheight = -1; // Initialization done only to prevent valgrind false positive, see https://github.com/bitcoin/bitcoin/pull/18785 if (!ParseInt32(height_str, &blockheight) || blockheight < 0) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid height: " + SanitizeString(height_str)); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 827b83d67e..f7ccbae706 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2242,8 +2242,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) { RPCHelpMan{ "dumptxoutset", - "\nWrite the serialized UTXO set to disk.\n" - "Incidentally flushes the latest coinsdb (leveldb) to disk.\n", + "\nWrite the serialized UTXO set to disk.\n", { {"path", RPCArg::Type::STR, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 5d0f2fd703..3045a74d7a 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -131,6 +131,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "importpubkey", 2, "rescan" }, { "importmulti", 0, "requests" }, { "importmulti", 1, "options" }, + { "importdescriptors", 0, "requests" }, { "verifychain", 0, "checklevel" }, { "verifychain", 1, "nblocks" }, { "getblockstats", 0, "hash_or_height" }, @@ -170,6 +171,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createwallet", 1, "disable_private_keys"}, { "createwallet", 2, "blank"}, { "createwallet", 4, "avoid_reuse"}, + { "createwallet", 5, "descriptors"}, { "getnodeaddresses", 0, "count"}, { "stop", 0, "wait" }, }; diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 68d83299ee..5279f40506 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -575,6 +575,7 @@ public: default: return nullopt; } } + bool IsSingleType() const final { return true; } }; /** A parsed raw(H) descriptor. */ @@ -602,6 +603,7 @@ public: default: return nullopt; } } + bool IsSingleType() const final { return true; } }; /** A parsed pk(P) descriptor. */ @@ -611,6 +613,7 @@ protected: std::vector<CScript> MakeScripts(const std::vector<CPubKey>& keys, const CScript*, FlatSigningProvider&) const override { return Vector(GetScriptForRawPubKey(keys[0])); } public: PKDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "pk") {} + bool IsSingleType() const final { return true; } }; /** A parsed pkh(P) descriptor. */ @@ -626,6 +629,7 @@ protected: public: PKHDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "pkh") {} Optional<OutputType> GetOutputType() const override { return OutputType::LEGACY; } + bool IsSingleType() const final { return true; } }; /** A parsed wpkh(P) descriptor. */ @@ -641,6 +645,7 @@ protected: public: WPKHDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "wpkh") {} Optional<OutputType> GetOutputType() const override { return OutputType::BECH32; } + bool IsSingleType() const final { return true; } }; /** A parsed combo(P) descriptor. */ @@ -664,6 +669,7 @@ protected: } public: ComboDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "combo") {} + bool IsSingleType() const final { return false; } }; /** A parsed multi(...) or sortedmulti(...) descriptor */ @@ -683,6 +689,7 @@ protected: } public: MultisigDescriptor(int threshold, std::vector<std::unique_ptr<PubkeyProvider>> providers, bool sorted = false) : DescriptorImpl(std::move(providers), {}, sorted ? "sortedmulti" : "multi"), m_threshold(threshold), m_sorted(sorted) {} + bool IsSingleType() const final { return true; } }; /** A parsed sh(...) descriptor. */ @@ -699,6 +706,7 @@ public: if (m_subdescriptor_arg->GetOutputType() == OutputType::BECH32) return OutputType::P2SH_SEGWIT; return OutputType::LEGACY; } + bool IsSingleType() const final { return true; } }; /** A parsed wsh(...) descriptor. */ @@ -709,6 +717,7 @@ protected: public: WSHDescriptor(std::unique_ptr<DescriptorImpl> desc) : DescriptorImpl({}, std::move(desc), "wsh") {} Optional<OutputType> GetOutputType() const override { return OutputType::BECH32; } + bool IsSingleType() const final { return true; } }; //////////////////////////////////////////////////////////////////////////// diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 69850eb670..17b43e7c81 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -87,6 +87,9 @@ struct Descriptor { /** Convert the descriptor back to a string, undoing parsing. */ virtual std::string ToString() const = 0; + /** Whether this descriptor will return one scriptPubKey or multiple (aka is or is not combo) */ + virtual bool IsSingleType() const = 0; + /** Convert the descriptor to a private string. This fails if the provided provider does not have the relevant private keys. */ virtual bool ToPrivateString(const SigningProvider& provider, std::string& out) const = 0; diff --git a/src/script/script.h b/src/script/script.h index daf4224530..773ffbb985 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -329,7 +329,7 @@ public: std::vector<unsigned char> result; const bool neg = value < 0; - uint64_t absvalue = neg ? -value : value; + uint64_t absvalue = neg ? ~static_cast<uint64_t>(value) + 1 : static_cast<uint64_t>(value); while(absvalue) { diff --git a/src/span.h b/src/span.h index 664ea8d4c6..73b37e35f3 100644 --- a/src/span.h +++ b/src/span.h @@ -25,6 +25,23 @@ public: constexpr Span(C* data, std::ptrdiff_t size) noexcept : m_data(data), m_size(size) {} constexpr Span(C* data, C* end) noexcept : m_data(data), m_size(end - data) {} + /** Implicit conversion of spans between compatible types. + * + * Specifically, if a pointer to an array of type O can be implicitly converted to a pointer to an array of type + * C, then permit implicit conversion of Span<O> to Span<C>. This matches the behavior of the corresponding + * C++20 std::span constructor. + * + * For example this means that a Span<T> can be converted into a Span<const T>. + */ + template <typename O, typename std::enable_if<std::is_convertible<O (*)[], C (*)[]>::value, int>::type = 0> + constexpr Span(const Span<O>& other) noexcept : m_data(other.m_data), m_size(other.m_size) {} + + /** Default copy constructor. */ + constexpr Span(const Span&) noexcept = default; + + /** Default assignment operator. */ + Span& operator=(const Span& other) noexcept = default; + constexpr C* data() const noexcept { return m_data; } constexpr C* begin() const noexcept { return m_data; } constexpr C* end() const noexcept { return m_data + m_size; } @@ -44,6 +61,8 @@ public: friend constexpr bool operator<=(const Span& a, const Span& b) noexcept { return !(b < a); } friend constexpr bool operator>(const Span& a, const Span& b) noexcept { return (b < a); } friend constexpr bool operator>=(const Span& a, const Span& b) noexcept { return !(a < b); } + + template <typename O> friend class Span; }; /** Create a span to a container exposing data() and size(). diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 9b14637b95..f17b539e09 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -253,8 +253,10 @@ void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) } if (addr) { *lockingSuccess = mlock(addr, len) == 0; -#ifdef MADV_DONTDUMP +#if defined(MADV_DONTDUMP) // Linux madvise(addr, len, MADV_DONTDUMP); +#elif defined(MADV_NOCORE) // FreeBSD + madvise(addr, len, MADV_NOCORE); #endif } return addr; diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index 090994263e..f29acace23 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -8,6 +8,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/fees.h> #include <cstdint> #include <string> @@ -23,4 +24,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) const CAmount rounded_fee = fee_filter_rounder.round(current_minimum_fee); assert(MoneyRange(rounded_fee)); } + const FeeReason fee_reason = fuzzed_data_provider.PickValueInArray({FeeReason::NONE, FeeReason::HALF_ESTIMATE, FeeReason::FULL_ESTIMATE, FeeReason::DOUBLE_ESTIMATE, FeeReason::CONSERVATIVE, FeeReason::MEMPOOL_MIN, FeeReason::PAYTXFEE, FeeReason::FALLBACK, FeeReason::REQUIRED}); + (void)StringForFeeReason(fee_reason); } diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 9dbf0fcc90..35d6804d4f 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -24,8 +24,8 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> -#include <time.h> #include <uint256.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/strencodings.h> #include <util/string.h> @@ -35,6 +35,7 @@ #include <cassert> #include <chrono> +#include <ctime> #include <limits> #include <set> #include <vector> @@ -147,11 +148,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) const CScriptNum script_num{i64}; (void)script_num.getint(); - // Avoid negation failure: - // script/script.h:332:35: runtime error: negation of -9223372036854775808 cannot be represented in type 'int64_t' (aka 'long'); cast to an unsigned type to negate this value to itself - if (script_num != CScriptNum{std::numeric_limits<int64_t>::min()}) { - (void)script_num.getvch(); - } + (void)script_num.getvch(); const arith_uint256 au256 = UintToArith256(u256); assert(ArithToUint256(au256) == u256); @@ -287,8 +284,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) try { const uint64_t deserialized_u64 = ReadCompactSize(stream); assert(u64 == deserialized_u64 && stream.empty()); + } catch (const std::ios_base::failure&) { } - catch (const std::ios_base::failure&) { - } + } + + try { + CHECK_NONFATAL(b); + } catch (const NonFatalCheckError&) { } } diff --git a/src/test/fuzz/kitchen_sink.cpp b/src/test/fuzz/kitchen_sink.cpp new file mode 100644 index 0000000000..af6dc71322 --- /dev/null +++ b/src/test/fuzz/kitchen_sink.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <rpc/util.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/error.h> + +#include <cstdint> +#include <vector> + +// The fuzzing kitchen sink: Fuzzing harness for functions that need to be +// fuzzed but a.) don't belong in any existing fuzzing harness file, and +// b.) are not important enough to warrant their own fuzzing harness file. +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const TransactionError transaction_error = fuzzed_data_provider.PickValueInArray<TransactionError>({TransactionError::OK, TransactionError::MISSING_INPUTS, TransactionError::ALREADY_IN_CHAIN, TransactionError::P2P_DISABLED, TransactionError::MEMPOOL_REJECTED, TransactionError::MEMPOOL_ERROR, TransactionError::INVALID_PSBT, TransactionError::PSBT_MISMATCH, TransactionError::SIGHASH_MISMATCH, TransactionError::MAX_FEE_EXCEEDED}); + (void)JSONRPCTransactionError(transaction_error); + (void)RPCErrorFromTransactionError(transaction_error); + (void)TransactionErrorString(transaction_error); +} diff --git a/src/test/fuzz/message.cpp b/src/test/fuzz/message.cpp new file mode 100644 index 0000000000..dfa98a812b --- /dev/null +++ b/src/test/fuzz/message.cpp @@ -0,0 +1,48 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <key_io.h> +#include <optional.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/message.h> +#include <util/strencodings.h> + +#include <cassert> +#include <cstdint> +#include <iostream> +#include <string> +#include <vector> + +void initialize() +{ + static const ECCVerifyHandle ecc_verify_handle; + ECC_Start(); + SelectParams(CBaseChainParams::REGTEST); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::string random_message = fuzzed_data_provider.ConsumeRandomLengthString(1024); + { + const std::vector<uint8_t> random_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); + CKey private_key; + private_key.Set(random_bytes.begin(), random_bytes.end(), fuzzed_data_provider.ConsumeBool()); + std::string signature; + const bool message_signed = MessageSign(private_key, random_message, signature); + if (private_key.IsValid()) { + assert(message_signed); + const MessageVerificationResult verification_result = MessageVerify(EncodeDestination(PKHash(private_key.GetPubKey().GetID())), signature, random_message); + assert(verification_result == MessageVerificationResult::OK); + } + } + { + (void)MessageHash(random_message); + (void)MessageVerify(fuzzed_data_provider.ConsumeRandomLengthString(1024), fuzzed_data_provider.ConsumeRandomLengthString(1024), random_message); + (void)SigningResultString(fuzzed_data_provider.PickValueInArray({SigningResult::OK, SigningResult::PRIVATE_KEY_NOT_AVAILABLE, SigningResult::SIGNING_FAILED})); + } +} diff --git a/src/test/fuzz/parse_hd_keypath.cpp b/src/test/fuzz/parse_hd_keypath.cpp index 9a23f4b2d4..f668ca8c48 100644 --- a/src/test/fuzz/parse_hd_keypath.cpp +++ b/src/test/fuzz/parse_hd_keypath.cpp @@ -2,12 +2,22 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> #include <util/bip32.h> +#include <cstdint> +#include <vector> + void test_one_input(const std::vector<uint8_t>& buffer) { const std::string keypath_str(buffer.begin(), buffer.end()); std::vector<uint32_t> keypath; (void)ParseHDKeypath(keypath_str, keypath); + + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::vector<uint32_t> random_keypath = ConsumeRandomLengthIntegralVector<uint32_t>(fuzzed_data_provider); + (void)FormatHDKeypath(random_keypath); + (void)WriteHDKeypath(random_keypath); } diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp new file mode 100644 index 0000000000..201f49c87b --- /dev/null +++ b/src/test/fuzz/policy_estimator.cpp @@ -0,0 +1,69 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <optional.h> +#include <policy/fees.h> +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <txmempool.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + CBlockPolicyEstimator block_policy_estimator; + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 3)) { + case 0: { + const Optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!mtx) { + break; + } + const CTransaction tx{*mtx}; + block_policy_estimator.processTransaction(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx), fuzzed_data_provider.ConsumeBool()); + if (fuzzed_data_provider.ConsumeBool()) { + (void)block_policy_estimator.removeTx(tx.GetHash(), /* inBlock */ fuzzed_data_provider.ConsumeBool()); + } + break; + } + case 1: { + std::vector<CTxMemPoolEntry> mempool_entries; + while (fuzzed_data_provider.ConsumeBool()) { + const Optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!mtx) { + break; + } + const CTransaction tx{*mtx}; + mempool_entries.push_back(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx)); + } + std::vector<const CTxMemPoolEntry*> ptrs; + ptrs.reserve(mempool_entries.size()); + for (const CTxMemPoolEntry& mempool_entry : mempool_entries) { + ptrs.push_back(&mempool_entry); + } + block_policy_estimator.processBlock(fuzzed_data_provider.ConsumeIntegral<unsigned int>(), ptrs); + break; + } + case 2: { + (void)block_policy_estimator.removeTx(ConsumeUInt256(fuzzed_data_provider), /* inBlock */ fuzzed_data_provider.ConsumeBool()); + break; + } + case 3: { + block_policy_estimator.FlushUnconfirmed(); + break; + } + } + (void)block_policy_estimator.estimateFee(fuzzed_data_provider.ConsumeIntegral<int>()); + EstimationResult result; + (void)block_policy_estimator.estimateRawFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeFloatingPoint<double>(), fuzzed_data_provider.PickValueInArray({FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE}), fuzzed_data_provider.ConsumeBool() ? &result : nullptr); + FeeCalculation fee_calculation; + (void)block_policy_estimator.estimateSmartFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeBool() ? &fee_calculation : nullptr, fuzzed_data_provider.ConsumeBool()); + (void)block_policy_estimator.HighestTargetTracked(fuzzed_data_provider.PickValueInArray({FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE})); + } +} diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp new file mode 100644 index 0000000000..eb54b05df9 --- /dev/null +++ b/src/test/fuzz/rbf.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <optional.h> +#include <policy/rbf.h> +#include <primitives/transaction.h> +#include <sync.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <txmempool.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + Optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!mtx) { + return; + } + CTxMemPool pool; + while (fuzzed_data_provider.ConsumeBool()) { + const Optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!another_mtx) { + break; + } + const CTransaction another_tx{*another_mtx}; + if (fuzzed_data_provider.ConsumeBool() && !mtx->vin.empty()) { + mtx->vin[0].prevout = COutPoint{another_tx.GetHash(), 0}; + } + LOCK2(cs_main, pool.cs); + pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, another_tx)); + } + const CTransaction tx{*mtx}; + if (fuzzed_data_provider.ConsumeBool()) { + LOCK2(cs_main, pool.cs); + pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx)); + } + { + LOCK(pool.cs); + (void)IsRBFOptIn(tx, pool); + } +} diff --git a/src/test/fuzz/scriptnum_ops.cpp b/src/test/fuzz/scriptnum_ops.cpp index 42b1432f13..f4e079fb89 100644 --- a/src/test/fuzz/scriptnum_ops.cpp +++ b/src/test/fuzz/scriptnum_ops.cpp @@ -129,10 +129,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } (void)script_num.getint(); - // Avoid negation failure: - // script/script.h:332:35: runtime error: negation of -9223372036854775808 cannot be represented in type 'int64_t' (aka 'long'); cast to an unsigned type to negate this value to itself - if (script_num != CScriptNum{std::numeric_limits<int64_t>::min()}) { - (void)script_num.getvch(); - } + (void)script_num.getvch(); } } diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 3de0cf8db7..49bee0e81f 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -115,4 +115,8 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(data_stream.empty()); assert(deserialized_string == random_string_1); } + { + int64_t amount_out; + (void)ParseFixedPoint(random_string_1, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1024), &amount_out); + } } diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp new file mode 100644 index 0000000000..7f378c2b13 --- /dev/null +++ b/src/test/fuzz/system.cpp @@ -0,0 +1,123 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/system.h> + +#include <cstdint> +#include <string> +#include <vector> + +namespace { +std::string GetArgumentName(const std::string& name) +{ + size_t idx = name.find('='); + if (idx == std::string::npos) { + idx = name.size(); + } + return name.substr(0, idx); +} +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + ArgsManager args_manager{}; + + if (fuzzed_data_provider.ConsumeBool()) { + SetupHelpOptions(args_manager); + } + + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 7)) { + case 0: { + args_manager.SelectConfigNetwork(fuzzed_data_provider.ConsumeRandomLengthString(16)); + break; + } + case 1: { + args_manager.SoftSetArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeRandomLengthString(16)); + break; + } + case 2: { + args_manager.ForceSetArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeRandomLengthString(16)); + break; + } + case 3: { + args_manager.SoftSetBoolArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeBool()); + break; + } + case 4: { + const OptionsCategory options_category = fuzzed_data_provider.PickValueInArray<OptionsCategory>({OptionsCategory::OPTIONS, OptionsCategory::CONNECTION, OptionsCategory::WALLET, OptionsCategory::WALLET_DEBUG_TEST, OptionsCategory::ZMQ, OptionsCategory::DEBUG_TEST, OptionsCategory::CHAINPARAMS, OptionsCategory::NODE_RELAY, OptionsCategory::BLOCK_CREATION, OptionsCategory::RPC, OptionsCategory::GUI, OptionsCategory::COMMANDS, OptionsCategory::REGISTER_COMMANDS, OptionsCategory::HIDDEN}); + // Avoid hitting: + // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + const std::string argument_name = GetArgumentName(fuzzed_data_provider.ConsumeRandomLengthString(16)); + if (args_manager.GetArgFlags(argument_name) != nullopt) { + break; + } + args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), options_category); + break; + } + case 5: { + // Avoid hitting: + // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + const std::vector<std::string> names = ConsumeRandomLengthStringVector(fuzzed_data_provider); + std::vector<std::string> hidden_arguments; + for (const std::string& name : names) { + const std::string hidden_argument = GetArgumentName(name); + if (args_manager.GetArgFlags(hidden_argument) != nullopt) { + continue; + } + if (std::find(hidden_arguments.begin(), hidden_arguments.end(), hidden_argument) != hidden_arguments.end()) { + continue; + } + hidden_arguments.push_back(hidden_argument); + } + args_manager.AddHiddenArgs(hidden_arguments); + break; + } + case 6: { + args_manager.ClearArgs(); + break; + } + case 7: { + const std::vector<std::string> random_arguments = ConsumeRandomLengthStringVector(fuzzed_data_provider); + std::vector<const char*> argv; + argv.resize(random_arguments.size()); + for (const std::string& random_argument : random_arguments) { + argv.push_back(random_argument.c_str()); + } + try { + std::string error; + (void)args_manager.ParseParameters(argv.size(), argv.data(), error); + } catch (const std::logic_error&) { + } + break; + } + } + } + + const std::string s1 = fuzzed_data_provider.ConsumeRandomLengthString(16); + const std::string s2 = fuzzed_data_provider.ConsumeRandomLengthString(16); + const int64_t i64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const bool b = fuzzed_data_provider.ConsumeBool(); + + (void)args_manager.GetArg(s1, i64); + (void)args_manager.GetArg(s1, s2); + (void)args_manager.GetArgFlags(s1); + (void)args_manager.GetArgs(s1); + (void)args_manager.GetBoolArg(s1, b); + try { + (void)args_manager.GetChainName(); + } catch (const std::runtime_error&) { + } + (void)args_manager.GetHelpMessage(); + (void)args_manager.GetUnrecognizedSections(); + (void)args_manager.GetUnsuitableSectionOnlyArgs(); + (void)args_manager.IsArgNegated(s1); + (void)args_manager.IsArgSet(s1); + + (void)HelpRequested(args_manager); +} diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 9c7b0d47a2..501bb1de5a 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -8,12 +8,15 @@ #include <amount.h> #include <arith_uint256.h> #include <attributes.h> +#include <consensus/consensus.h> #include <optional.h> +#include <primitives/transaction.h> #include <script/script.h> #include <serialize.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <txmempool.h> #include <uint256.h> #include <version.h> @@ -97,6 +100,21 @@ NODISCARD inline arith_uint256 ConsumeArithUInt256(FuzzedDataProvider& fuzzed_da return UintToArith256(ConsumeUInt256(fuzzed_data_provider)); } +NODISCARD inline CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept +{ + // Avoid: + // policy/feerate.cpp:28:34: runtime error: signed integer overflow: 34873208148477500 * 1000 cannot be represented in type 'long' + // + // Reproduce using CFeeRate(348732081484775, 10).GetFeePerK() + const CAmount fee = std::min<CAmount>(ConsumeMoney(fuzzed_data_provider), std::numeric_limits<CAmount>::max() / static_cast<CAmount>(100000)); + assert(MoneyRange(fee)); + const int64_t time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const unsigned int entry_height = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + const bool spends_coinbase = fuzzed_data_provider.ConsumeBool(); + const unsigned int sig_op_cost = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, MAX_BLOCK_SIGOPS_COST); + return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; +} + template <typename T> NODISCARD bool MultiplicationOverflow(const T i, const T j) noexcept { diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 4d50845256..9a490aaf6b 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -14,7 +14,7 @@ BOOST_AUTO_TEST_CASE(basic_sanity) { BOOST_CHECK_MESSAGE(glibc_sanity_test() == true, "libc sanity test"); BOOST_CHECK_MESSAGE(glibcxx_sanity_test() == true, "stdlib sanity test"); - BOOST_CHECK_MESSAGE(ECC_InitSanityCheck() == true, "openssl ECC test"); + BOOST_CHECK_MESSAGE(ECC_InitSanityCheck() == true, "secp256k1 sanity test"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 9d0ae56c3f..ddbc68f8e2 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -305,7 +305,6 @@ BOOST_AUTO_TEST_CASE(test_Get) t1.vout[0].scriptPubKey << OP_1; BOOST_CHECK(AreInputsStandard(CTransaction(t1), coins)); - BOOST_CHECK_EQUAL(coins.GetValueIn(CTransaction(t1)), (50+21+22)*CENT); } static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const CScript& outscript, CTransactionRef& output, CMutableTransaction& input, bool success = true) diff --git a/src/test/util/logging.cpp b/src/test/util/logging.cpp index fe2e69104b..65a64f2384 100644 --- a/src/test/util/logging.cpp +++ b/src/test/util/logging.cpp @@ -11,13 +11,13 @@ #include <stdexcept> -DebugLogHelper::DebugLogHelper(std::string message) - : m_message{std::move(message)} +DebugLogHelper::DebugLogHelper(std::string message, MatchFn match) + : m_message{std::move(message)}, m_match(std::move(match)) { m_print_connection = LogInstance().PushBackCallback( [this](const std::string& s) { if (m_found) return; - m_found = s.find(m_message) != std::string::npos; + m_found = s.find(m_message) != std::string::npos && m_match(&s); }); noui_test_redirect(); } @@ -26,7 +26,7 @@ void DebugLogHelper::check_found() { noui_reconnect(); LogInstance().DeleteCallback(m_print_connection); - if (!m_found) { + if (!m_found && m_match(nullptr)) { throw std::runtime_error(strprintf("'%s' not found in debug log\n", m_message)); } } diff --git a/src/test/util/logging.h b/src/test/util/logging.h index 45ec44173c..1fcf7ca305 100644 --- a/src/test/util/logging.h +++ b/src/test/util/logging.h @@ -17,10 +17,22 @@ class DebugLogHelper bool m_found{false}; std::list<std::function<void(const std::string&)>>::iterator m_print_connection; + //! Custom match checking function. + //! + //! Invoked with pointers to lines containing matching strings, and with + //! null if check_found() is called without any successful match. + //! + //! Can return true to enable default DebugLogHelper behavior of: + //! (1) ending search after first successful match, and + //! (2) raising an error in check_found if no match was found + //! Can return false to do the opposite in either case. + using MatchFn = std::function<bool(const std::string* line)>; + MatchFn m_match; + void check_found(); public: - DebugLogHelper(std::string message); + DebugLogHelper(std::string message, MatchFn match = [](const std::string*){ return true; }); ~DebugLogHelper() { check_found(); } }; diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index b8af869b8e..c345f1eafb 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -340,4 +340,38 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) rpc_thread.join(); } } + +BOOST_AUTO_TEST_CASE(witness_commitment_index) +{ + CScript pubKey; + pubKey << 1 << OP_TRUE; + auto ptemplate = BlockAssembler(*m_node.mempool, Params()).CreateNewBlock(pubKey); + CBlock pblock = ptemplate->block; + + CTxOut witness; + witness.scriptPubKey.resize(MINIMUM_WITNESS_COMMITMENT); + witness.scriptPubKey[0] = OP_RETURN; + witness.scriptPubKey[1] = 0x24; + witness.scriptPubKey[2] = 0xaa; + witness.scriptPubKey[3] = 0x21; + witness.scriptPubKey[4] = 0xa9; + witness.scriptPubKey[5] = 0xed; + + // A witness larger than the minimum size is still valid + CTxOut min_plus_one = witness; + min_plus_one.scriptPubKey.resize(MINIMUM_WITNESS_COMMITMENT + 1); + + CTxOut invalid = witness; + invalid.scriptPubKey[0] = OP_VERIFY; + + CMutableTransaction txCoinbase(*pblock.vtx[0]); + txCoinbase.vout.resize(4); + txCoinbase.vout[0] = witness; + txCoinbase.vout[1] = witness; + txCoinbase.vout[2] = min_plus_one; + txCoinbase.vout[3] = invalid; + pblock.vtx[0] = MakeTransactionRef(std::move(txCoinbase)); + + BOOST_CHECK_EQUAL(GetWitnessCommitmentIndex(pblock), 2); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 102e50dc5c..c5c0208d8f 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -417,6 +417,8 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) for (const CTxIn& txin : it->GetTx().vin) mapNextTx.erase(txin.prevout); + RemoveUnbroadcastTx(hash, true /* add logging because unchecked */ ); + if (vTxHashes.size() > 1) { vTxHashes[it->vTxHashesIdx] = std::move(vTxHashes.back()); vTxHashes[it->vTxHashesIdx].second->vTxHashesIdx = it->vTxHashesIdx; @@ -919,6 +921,15 @@ size_t CTxMemPool::DynamicMemoryUsage() const { return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 12 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; } +void CTxMemPool::RemoveUnbroadcastTx(const uint256& txid, const bool unchecked) { + LOCK(cs); + + if (m_unbroadcast_txids.erase(txid)) + { + LogPrint(BCLog::MEMPOOL, "Removed %i from set of unbroadcast txns%s\n", txid.GetHex(), (unchecked ? " before confirmation that txn was sent out" : "")); + } +} + void CTxMemPool::RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason) { AssertLockHeld(cs); UpdateForRemoveFromMempool(stage, updateDescendants); diff --git a/src/txmempool.h b/src/txmempool.h index 3dae0a04c7..4bee78b8d6 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -549,6 +549,9 @@ private: std::vector<indexed_transaction_set::const_iterator> GetSortedDepthAndScore() const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** track locally submitted transactions to periodically retry initial broadcast */ + std::set<uint256> m_unbroadcast_txids GUARDED_BY(cs); + public: indirectmap<COutPoint, const CTransaction*> mapNextTx GUARDED_BY(cs); std::map<uint256, CAmount> mapDeltas; @@ -698,6 +701,21 @@ public: size_t DynamicMemoryUsage() const; + /** Adds a transaction to the unbroadcast set */ + void AddUnbroadcastTx(const uint256& txid) { + LOCK(cs); + m_unbroadcast_txids.insert(txid); + } + + /** Removes a transaction from the unbroadcast set */ + void RemoveUnbroadcastTx(const uint256& txid, const bool unchecked = false); + + /** Returns transactions in unbroadcast set */ + const std::set<uint256> GetUnbroadcastTxs() const { + LOCK(cs); + return m_unbroadcast_txids; + } + private: /** UpdateForDescendants is used by UpdateTransactionsFromBlock to update * the descendants for a single transaction that has been added to the diff --git a/src/util/system.cpp b/src/util/system.cpp index 0790ea0d48..2013b416db 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -17,7 +17,7 @@ #endif #ifndef WIN32 -// for posix_fallocate +// for posix_fallocate, in configure.ac we check if it is present after this #ifdef __linux__ #ifdef _POSIX_C_SOURCE @@ -1019,7 +1019,7 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) { } ftruncate(fileno(file), static_cast<off_t>(offset) + length); #else - #if defined(__linux__) + #if defined(HAVE_POSIX_FALLOCATE) // Version using posix_fallocate off_t nEndPos = (off_t)offset + length; if (0 == posix_fallocate(fileno(file), 0, nEndPos)) return; diff --git a/src/util/translation.h b/src/util/translation.h index fc45da440a..b2d964c977 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -18,6 +18,20 @@ struct bilingual_str { std::string translated; }; +inline bilingual_str operator+(const bilingual_str& lhs, const bilingual_str& rhs) +{ + return bilingual_str{ + lhs.original + rhs.original, + lhs.translated + rhs.translated}; +} + +/** Mark a bilingual_str as untranslated */ +inline static bilingual_str Untranslated(std::string original) { return {original, original}; } +/** Unary operator to return the original */ +inline static std::string OpOriginal(const bilingual_str& b) { return b.original; } +/** Unary operator to return the translation */ +inline static std::string OpTranslated(const bilingual_str& b) { return b.translated; } + namespace tinyformat { template <typename... Args> bilingual_str format(const bilingual_str& fmt, const Args&... args) diff --git a/src/validation.cpp b/src/validation.cpp index 25975e3e31..a924af90af 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2279,11 +2279,11 @@ bool CChainState::FlushStateToDisk( LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { if (nManualPruneHeight > 0) { - LOG_TIME_MILLIS("find files to prune (manual)", BCLog::BENCH); + LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCH); FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight); } else { - LOG_TIME_MILLIS("find files to prune", BCLog::BENCH); + LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight()); fCheckForPruning = false; @@ -2321,7 +2321,7 @@ bool CChainState::FlushStateToDisk( return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); } { - LOG_TIME_MILLIS("write block and undo data to disk", BCLog::BENCH); + LOG_TIME_MILLIS_WITH_CATEGORY("write block and undo data to disk", BCLog::BENCH); // First make sure all block and undo data is flushed to disk. FlushBlockFile(); @@ -2329,7 +2329,7 @@ bool CChainState::FlushStateToDisk( // Then update all block file information (which may refer to block and undo files). { - LOG_TIME_MILLIS("write block index to disk", BCLog::BENCH); + LOG_TIME_MILLIS_WITH_CATEGORY("write block index to disk", BCLog::BENCH); std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; vFiles.reserve(setDirtyFileInfo.size()); @@ -2349,7 +2349,7 @@ bool CChainState::FlushStateToDisk( } // Finally remove any pruned files if (fFlushForPrune) { - LOG_TIME_MILLIS("unlink pruned files", BCLog::BENCH); + LOG_TIME_MILLIS_WITH_CATEGORY("unlink pruned files", BCLog::BENCH); UnlinkPrunedFiles(setFilesToPrune); } @@ -3386,7 +3386,14 @@ int GetWitnessCommitmentIndex(const CBlock& block) int commitpos = -1; if (!block.vtx.empty()) { for (size_t o = 0; o < block.vtx[0]->vout.size(); o++) { - if (block.vtx[0]->vout[o].scriptPubKey.size() >= 38 && block.vtx[0]->vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0]->vout[o].scriptPubKey[1] == 0x24 && block.vtx[0]->vout[o].scriptPubKey[2] == 0xaa && block.vtx[0]->vout[o].scriptPubKey[3] == 0x21 && block.vtx[0]->vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0]->vout[o].scriptPubKey[5] == 0xed) { + const CTxOut& vout = block.vtx[0]->vout[o]; + if (vout.scriptPubKey.size() >= MINIMUM_WITNESS_COMMITMENT && + vout.scriptPubKey[0] == OP_RETURN && + vout.scriptPubKey[1] == 0x24 && + vout.scriptPubKey[2] == 0xaa && + vout.scriptPubKey[3] == 0x21 && + vout.scriptPubKey[4] == 0xa9 && + vout.scriptPubKey[5] == 0xed) { commitpos = o; } } @@ -3417,7 +3424,7 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc CHash256().Write(witnessroot.begin(), 32).Write(ret.data(), 32).Finalize(witnessroot.begin()); CTxOut out; out.nValue = 0; - out.scriptPubKey.resize(38); + out.scriptPubKey.resize(MINIMUM_WITNESS_COMMITMENT); out.scriptPubKey[0] = OP_RETURN; out.scriptPubKey[1] = 0x24; out.scriptPubKey[2] = 0xaa; @@ -4637,7 +4644,7 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return ::ChainstateActive().LoadGenesisBlock(chainparams); } -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp) +void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; @@ -4649,7 +4656,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SERIALIZED_SIZE, MAX_BLOCK_SERIALIZED_SIZE+8, SER_DISK, CLIENT_VERSION); uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { - boost::this_thread::interruption_point(); + if (ShutdownRequested()) return; blkdat.SetPos(nRewind); nRewind++; // start one byte further next time, in case of failure @@ -4754,9 +4761,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi } catch (const std::runtime_error& e) { AbortNode(std::string("System error: ") + e.what()); } - if (nLoaded > 0) - LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, GetTimeMillis() - nStart); - return nLoaded > 0; + LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, GetTimeMillis() - nStart); } void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) @@ -4998,6 +5003,7 @@ bool LoadMempool(CTxMemPool& pool) int64_t expired = 0; int64_t failed = 0; int64_t already_there = 0; + int64_t unbroadcast = 0; int64_t nNow = GetTime(); try { @@ -5051,12 +5057,21 @@ bool LoadMempool(CTxMemPool& pool) for (const auto& i : mapDeltas) { pool.PrioritiseTransaction(i.first, i.second); } + + std::set<uint256> unbroadcast_txids; + file >> unbroadcast_txids; + unbroadcast = unbroadcast_txids.size(); + + for (const auto& txid : unbroadcast_txids) { + pool.AddUnbroadcastTx(txid); + } + } catch (const std::exception& e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); return false; } - LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there\n", count, failed, expired, already_there); + LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); return true; } @@ -5066,6 +5081,7 @@ bool DumpMempool(const CTxMemPool& pool) std::map<uint256, CAmount> mapDeltas; std::vector<TxMempoolInfo> vinfo; + std::set<uint256> unbroadcast_txids; static Mutex dump_mutex; LOCK(dump_mutex); @@ -5076,6 +5092,7 @@ bool DumpMempool(const CTxMemPool& pool) mapDeltas[i.first] = i.second; } vinfo = pool.infoAll(); + unbroadcast_txids = pool.GetUnbroadcastTxs(); } int64_t mid = GetTimeMicros(); @@ -5100,6 +5117,10 @@ bool DumpMempool(const CTxMemPool& pool) } file << mapDeltas; + + LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); + file << unbroadcast_txids; + if (!FileCommit(file.Get())) throw std::runtime_error("FileCommit failed"); file.fclose(); diff --git a/src/validation.h b/src/validation.h index 91b1ba6497..c4a5cc4593 100644 --- a/src/validation.h +++ b/src/validation.h @@ -92,6 +92,8 @@ static const unsigned int DEFAULT_CHECKLEVEL = 3; // one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB // Setting the target to >= 550 MiB will make it likely we can respect the target. static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; +/** Minimum size of a witness commitment structure. Defined in BIP 141. **/ +static constexpr size_t MINIMUM_WITNESS_COMMITMENT{38}; struct BlockHasher { @@ -180,7 +182,7 @@ FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ fs::path GetBlockPosFilename(const FlatFilePos &pos); /** Import blocks from an external file */ -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp = nullptr); +void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); /** Load the block tree and coins database from disk, diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 67be4d85d2..3f7c2d09cc 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -393,7 +393,7 @@ bool BerkeleyBatch::Recover(const fs::path& file_path, void *callbackDataIn, boo return fSuccess; } -bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, std::string& errorStr) +bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr) { std::string walletFile; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); @@ -403,14 +403,14 @@ bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, std::string& er LogPrintf("Using wallet %s\n", file_path.string()); if (!env->Open(true /* retry */)) { - errorStr = strprintf(_("Error initializing wallet database environment %s!").translated, walletDir); + errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir); return false; } return true; } -bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::vector<std::string>& warnings, std::string& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc) +bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::vector<bilingual_str>& warnings, bilingual_str& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc) { std::string walletFile; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); @@ -425,12 +425,12 @@ bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::vector<st warnings.push_back(strprintf(_("Warning: Wallet file corrupt, data salvaged!" " Original %s saved as %s in %s; if" " your balance or transactions are incorrect you should" - " restore from a backup.").translated, + " restore from a backup."), walletFile, backup_filename, walletDir)); } if (r == BerkeleyEnvironment::VerifyResult::RECOVER_FAIL) { - errorStr = strprintf(_("%s corrupt, salvage failed").translated, walletFile); + errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile); return false; } } diff --git a/src/wallet/db.h b/src/wallet/db.h index 1c5f42abca..1bf3375475 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -21,6 +21,8 @@ #include <db_cxx.h> +struct bilingual_str; + static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; static const bool DEFAULT_WALLET_PRIVDB = true; @@ -242,9 +244,9 @@ public: ideal to be called periodically */ static bool PeriodicFlush(BerkeleyDatabase& database); /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path& file_path, std::string& errorStr); + static bool VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& file_path, std::vector<std::string>& warnings, std::string& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc); + static bool VerifyDatabaseFile(const fs::path& file_path, std::vector<bilingual_str>& warnings, bilingual_str& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc); template <typename K, typename T> bool Read(const K& key, T& value) diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index dafa022ded..cacf306891 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -3,44 +3,45 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <interfaces/chain.h> -#include <wallet/coincontrol.h> -#include <wallet/feebumper.h> -#include <wallet/fees.h> -#include <wallet/wallet.h> #include <policy/fees.h> #include <policy/policy.h> #include <util/moneystr.h> #include <util/rbf.h> #include <util/system.h> +#include <util/translation.h> +#include <wallet/coincontrol.h> +#include <wallet/feebumper.h> +#include <wallet/fees.h> +#include <wallet/wallet.h> //! Check whether transaction has descendant in wallet or mempool, or has been //! mined, or conflicts with a mined transaction. Return a feebumper::Result. -static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWalletTx& wtx, std::vector<std::string>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWalletTx& wtx, std::vector<bilingual_str>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { if (wallet.HasWalletSpend(wtx.GetHash())) { - errors.push_back("Transaction has descendants in the wallet"); + errors.push_back(Untranslated("Transaction has descendants in the wallet")); return feebumper::Result::INVALID_PARAMETER; } { if (wallet.chain().hasDescendantsInMempool(wtx.GetHash())) { - errors.push_back("Transaction has descendants in the mempool"); + errors.push_back(Untranslated("Transaction has descendants in the mempool")); return feebumper::Result::INVALID_PARAMETER; } } if (wtx.GetDepthInMainChain() != 0) { - errors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); + errors.push_back(Untranslated("Transaction has been mined, or is conflicted with a mined transaction")); return feebumper::Result::WALLET_ERROR; } if (!SignalsOptInRBF(*wtx.tx)) { - errors.push_back("Transaction is not BIP 125 replaceable"); + errors.push_back(Untranslated("Transaction is not BIP 125 replaceable")); return feebumper::Result::WALLET_ERROR; } if (wtx.mapValue.count("replaced_by_txid")) { - errors.push_back(strprintf("Cannot bump transaction %s which was already bumped by transaction %s", wtx.GetHash().ToString(), wtx.mapValue.at("replaced_by_txid"))); + errors.push_back(strprintf(Untranslated("Cannot bump transaction %s which was already bumped by transaction %s"), wtx.GetHash().ToString(), wtx.mapValue.at("replaced_by_txid"))); return feebumper::Result::WALLET_ERROR; } @@ -48,7 +49,7 @@ static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWallet // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; if (!wallet.IsAllFromMe(*wtx.tx, filter)) { - errors.push_back("Transaction contains inputs that don't belong to this wallet"); + errors.push_back(Untranslated("Transaction contains inputs that don't belong to this wallet")); return feebumper::Result::WALLET_ERROR; } @@ -57,7 +58,8 @@ static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWallet } //! Check if the user provided a valid feeRate -static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wtx, const CFeeRate& newFeerate, const int64_t maxTxSize, std::vector<std::string>& errors) { +static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wtx, const CFeeRate& newFeerate, const int64_t maxTxSize, std::vector<bilingual_str>& errors) +{ // check that fee rate is higher than mempool's minimum fee // (no point in bumping fee if we know that the new tx won't be accepted to the mempool) // This may occur if the user set fee_rate or paytxfee too low, if fallbackfee is too low, or, perhaps, @@ -67,7 +69,7 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt if (newFeerate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { errors.push_back(strprintf( - "New fee rate (%s) is lower than the minimum fee rate (%s) to get into the mempool -- ", + Untranslated("New fee rate (%s) is lower than the minimum fee rate (%s) to get into the mempool -- "), FormatMoney(newFeerate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()))); return feebumper::Result::WALLET_ERROR; @@ -86,14 +88,14 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt CAmount minTotalFee = nOldFeeRate.GetFee(maxTxSize) + incrementalRelayFee.GetFee(maxTxSize); if (new_total_fee < minTotalFee) { - errors.push_back(strprintf("Insufficient total fee %s, must be at least %s (oldFee %s + incrementalFee %s)", + errors.push_back(strprintf(Untranslated("Insufficient total fee %s, must be at least %s (oldFee %s + incrementalFee %s)"), FormatMoney(new_total_fee), FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxTxSize)), FormatMoney(incrementalRelayFee.GetFee(maxTxSize)))); return feebumper::Result::INVALID_PARAMETER; } CAmount requiredFee = GetRequiredFee(wallet, maxTxSize); if (new_total_fee < requiredFee) { - errors.push_back(strprintf("Insufficient total fee (cannot be less than required fee %s)", + errors.push_back(strprintf(Untranslated("Insufficient total fee (cannot be less than required fee %s)"), FormatMoney(requiredFee))); return feebumper::Result::INVALID_PARAMETER; } @@ -101,8 +103,8 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt // Check that in all cases the new fee doesn't violate maxTxFee const CAmount max_tx_fee = wallet.m_default_max_tx_fee; if (new_total_fee > max_tx_fee) { - errors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than -maxtxfee %s)", - FormatMoney(new_total_fee), FormatMoney(max_tx_fee))); + errors.push_back(strprintf(Untranslated("Specified or calculated fee %s is too high (cannot be higher than -maxtxfee %s)"), + FormatMoney(new_total_fee), FormatMoney(max_tx_fee))); return feebumper::Result::WALLET_ERROR; } @@ -140,28 +142,26 @@ namespace feebumper { bool TransactionCanBeBumped(const CWallet& wallet, const uint256& txid) { - auto locked_chain = wallet.chain().lock(); LOCK(wallet.cs_wallet); const CWalletTx* wtx = wallet.GetWalletTx(txid); if (wtx == nullptr) return false; - std::vector<std::string> errors_dummy; + std::vector<bilingual_str> errors_dummy; feebumper::Result res = PreconditionChecks(wallet, *wtx, errors_dummy); return res == feebumper::Result::OK; } -Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCoinControl& coin_control, std::vector<std::string>& errors, +Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCoinControl& coin_control, std::vector<bilingual_str>& errors, CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) { // We are going to modify coin control later, copy to re-use CCoinControl new_coin_control(coin_control); - auto locked_chain = wallet.chain().lock(); LOCK(wallet.cs_wallet); errors.clear(); auto it = wallet.mapWallet.find(txid); if (it == wallet.mapWallet.end()) { - errors.push_back("Invalid or non-wallet transaction id"); + errors.push_back(Untranslated("Invalid or non-wallet transaction id")); return Result::INVALID_ADDRESS_OR_KEY; } const CWalletTx& wtx = it->second; @@ -218,9 +218,9 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo CTransactionRef tx_new = MakeTransactionRef(); CAmount fee_ret; int change_pos_in_out = -1; // No requested location for change - std::string fail_reason; - if (!wallet.CreateTransaction(*locked_chain, recipients, tx_new, fee_ret, change_pos_in_out, fail_reason, new_coin_control, false)) { - errors.push_back("Unable to create transaction: " + fail_reason); + bilingual_str fail_reason; + if (!wallet.CreateTransaction(recipients, tx_new, fee_ret, change_pos_in_out, fail_reason, new_coin_control, false)) { + errors.push_back(Untranslated("Unable to create transaction.") + Untranslated(" ") + fail_reason); return Result::WALLET_ERROR; } @@ -240,21 +240,19 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } bool SignTransaction(CWallet& wallet, CMutableTransaction& mtx) { - auto locked_chain = wallet.chain().lock(); LOCK(wallet.cs_wallet); return wallet.SignTransaction(mtx); } -Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransaction&& mtx, std::vector<std::string>& errors, uint256& bumped_txid) +Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransaction&& mtx, std::vector<bilingual_str>& errors, uint256& bumped_txid) { - auto locked_chain = wallet.chain().lock(); LOCK(wallet.cs_wallet); if (!errors.empty()) { return Result::MISC_ERROR; } auto it = txid.IsNull() ? wallet.mapWallet.end() : wallet.mapWallet.find(txid); if (it == wallet.mapWallet.end()) { - errors.push_back("Invalid or non-wallet transaction id"); + errors.push_back(Untranslated("Invalid or non-wallet transaction id")); return Result::MISC_ERROR; } CWalletTx& oldWtx = it->second; @@ -279,7 +277,7 @@ Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransacti // along with an exception. It would be good to return information about // wtxBumped to the caller even if marking the original transaction // replaced does not succeed for some reason. - errors.push_back("Created new bumpfee transaction but could not mark the original transaction as replaced"); + errors.push_back(Untranslated("Created new bumpfee transaction but could not mark the original transaction as replaced")); } return Result::OK; } diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index fc038ae731..50577c9d3e 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -12,6 +12,7 @@ class CWalletTx; class uint256; class CCoinControl; enum class FeeEstimateMode; +struct bilingual_str; namespace feebumper { @@ -30,12 +31,12 @@ bool TransactionCanBeBumped(const CWallet& wallet, const uint256& txid); //! Create bumpfee transaction based on feerate estimates. Result CreateRateBumpTransaction(CWallet& wallet, - const uint256& txid, - const CCoinControl& coin_control, - std::vector<std::string>& errors, - CAmount& old_fee, - CAmount& new_fee, - CMutableTransaction& mtx); + const uint256& txid, + const CCoinControl& coin_control, + std::vector<bilingual_str>& errors, + CAmount& old_fee, + CAmount& new_fee, + CMutableTransaction& mtx); //! Sign the new transaction, //! @return false if the tx couldn't be found or if it was @@ -47,10 +48,10 @@ bool SignTransaction(CWallet& wallet, CMutableTransaction& mtx); //! but sets errors if the tx could not be added to the mempool (will try later) //! or if the old transaction could not be marked as replaced. Result CommitTransaction(CWallet& wallet, - const uint256& txid, - CMutableTransaction&& mtx, - std::vector<std::string>& errors, - uint256& bumped_txid); + const uint256& txid, + CMutableTransaction&& mtx, + std::vector<bilingual_str>& errors, + uint256& bumped_txid); } // namespace feebumper diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 0afd2dfcf0..217a950457 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -53,12 +53,14 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal return false; } - std::string error_string; - std::vector<std::string> warnings; + bilingual_str error_string; + std::vector<bilingual_str> warnings; bool verify_success = CWallet::Verify(chain, location, salvage_wallet, error_string, warnings); - if (!error_string.empty()) chain.initError(error_string); - if (!warnings.empty()) chain.initWarning(Join(warnings, "\n")); - if (!verify_success) return false; + if (!warnings.empty()) chain.initWarning(Join(warnings, "\n", OpTranslated)); + if (!verify_success) { + chain.initError(error_string.translated); + return false; + } } return true; @@ -68,12 +70,12 @@ bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& walle { try { for (const std::string& walletFile : wallet_files) { - std::string error; - std::vector<std::string> warnings; + bilingual_str error; + std::vector<bilingual_str> warnings; std::shared_ptr<CWallet> pwallet = CWallet::CreateWalletFromFile(chain, WalletLocation(walletFile), error, warnings); - if (!warnings.empty()) chain.initWarning(Join(warnings, "\n")); + if (!warnings.empty()) chain.initWarning(Join(warnings, "\n", OpTranslated)); if (!pwallet) { - chain.initError(error); + chain.initError(error.translated); return false; } AddWallet(pwallet); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 86e4e06673..c863d22530 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -133,7 +133,6 @@ UniValue importprivkey(const JSONRPCRequest& request) WalletRescanReserver reserver(*pwallet); bool fRescan = true; { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -285,7 +284,6 @@ UniValue importaddress(const JSONRPCRequest& request) fP2SH = request.params[3].get_bool(); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); CTxDestination dest = DecodeDestination(request.params[0].get_str()); @@ -317,7 +315,6 @@ UniValue importaddress(const JSONRPCRequest& request) { RescanWallet(*pwallet, reserver); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); pwallet->ReacceptWalletTransactions(); } @@ -361,7 +358,6 @@ UniValue importprunedfunds(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Something wrong with merkleblock"); } - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); int height; if (!pwallet->chain().findAncestorByHash(pwallet->GetLastBlockHash(), merkleBlock.header.GetHash(), FoundBlock().height(height))) { @@ -407,7 +403,6 @@ UniValue removeprunedfunds(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); uint256 hash(ParseHashV(request.params[0], "txid")); @@ -487,7 +482,6 @@ UniValue importpubkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey is not a valid public key"); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); std::set<CScript> script_pub_keys; @@ -505,7 +499,6 @@ UniValue importpubkey(const JSONRPCRequest& request) { RescanWallet(*pwallet, reserver); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); pwallet->ReacceptWalletTransactions(); } @@ -557,7 +550,6 @@ UniValue importwallet(const JSONRPCRequest& request) int64_t nTimeBegin = 0; bool fGood = true; { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -700,7 +692,6 @@ UniValue dumpprivkey(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); - auto locked_chain = pwallet->chain().lock(); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(pwallet); @@ -735,7 +726,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) "Note that if your wallet contains keys which are not derived from your HD seed (e.g. imported keys), these are not covered by\n" "only backing up the seed itself, and must be backed up too (e.g. ensure you back up the whole dumpfile).\n", { - {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The filename with path (either absolute or relative to bitcoind)"}, + {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The filename with path (absolute path recommended)"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -756,7 +747,6 @@ UniValue dumpwallet(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(&wallet); @@ -780,7 +770,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) std::map<CKeyID, int64_t> mapKeyBirth; const std::map<CKeyID, int64_t>& mapKeyPool = spk_man.GetAllReserveKeys(); - pwallet->GetKeyBirthTimes(*locked_chain, mapKeyBirth); + pwallet->GetKeyBirthTimes(mapKeyBirth); std::set<CScriptID> scripts = spk_man.GetCScripts(); @@ -1379,7 +1369,6 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) int64_t nLowestTimestamp = 0; UniValue response(UniValue::VARR); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -1414,7 +1403,6 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) if (fRescan && fRunScan && requests.size()) { int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, reserver, true /* update */); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); pwallet->ReacceptWalletTransactions(); } @@ -1458,3 +1446,295 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) return response; } + +static UniValue ProcessDescriptorImport(CWallet * const pwallet, const UniValue& data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + UniValue warnings(UniValue::VARR); + UniValue result(UniValue::VOBJ); + + try { + if (!data.exists("desc")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Descriptor not found."); + } + + const std::string& descriptor = data["desc"].get_str(); + const bool active = data.exists("active") ? data["active"].get_bool() : false; + const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; + const std::string& label = data.exists("label") ? data["label"].get_str() : ""; + + // Parse descriptor string + FlatSigningProvider keys; + std::string error; + auto parsed_desc = Parse(descriptor, keys, error, /* require_checksum = */ true); + if (!parsed_desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); + } + + // Range check + int64_t range_start = 0, range_end = 1, next_index = 0; + if (!parsed_desc->IsRange() && data.exists("range")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for an un-ranged descriptor"); + } else if (parsed_desc->IsRange()) { + if (data.exists("range")) { + auto range = ParseDescriptorRange(data["range"]); + range_start = range.first; + range_end = range.second + 1; // Specified range end is inclusive, but we need range end as exclusive + } else { + warnings.push_back("Range not given, using default keypool range"); + range_start = 0; + range_end = gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE); + } + next_index = range_start; + + if (data.exists("next_index")) { + next_index = data["next_index"].get_int64(); + // bound checks + if (next_index < range_start || next_index >= range_end) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "next_index is out of range"); + } + } + } + + // Active descriptors must be ranged + if (active && !parsed_desc->IsRange()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Active descriptors must be ranged"); + } + + // Ranged descriptors should not have a label + if (data.exists("range") && data.exists("label")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptors should not have a label"); + } + + // Internal addresses should not have a label either + if (internal && data.exists("label")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal addresses should not have a label"); + } + + // Combo descriptor check + if (active && !parsed_desc->IsSingleType()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Combo descriptors cannot be set to active"); + } + + // If the wallet disabled private keys, abort if private keys exist + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !keys.keys.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); + } + + // Need to ExpandPrivate to check if private keys are available for all pubkeys + FlatSigningProvider expand_keys; + std::vector<CScript> scripts; + parsed_desc->Expand(0, keys, scripts, expand_keys); + parsed_desc->ExpandPrivate(0, keys, expand_keys); + + // Check if all private keys are provided + bool have_all_privkeys = !expand_keys.keys.empty(); + for (const auto& entry : expand_keys.origins) { + const CKeyID& key_id = entry.first; + CKey key; + if (!expand_keys.GetKey(key_id, key)) { + have_all_privkeys = false; + break; + } + } + + // If private keys are enabled, check some things. + if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (keys.keys.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import descriptor without private keys to a wallet with private keys enabled"); + } + if (!have_all_privkeys) { + warnings.push_back("Not all private keys provided. Some wallet functionality may return unexpected errors"); + } + } + + WalletDescriptor w_desc(std::move(parsed_desc), timestamp, range_start, range_end, next_index); + + // Check if the wallet already contains the descriptor + auto existing_spk_manager = pwallet->GetDescriptorScriptPubKeyMan(w_desc); + if (existing_spk_manager) { + LOCK(existing_spk_manager->cs_desc_man); + if (range_start > existing_spk_manager->GetWalletDescriptor().range_start) { + throw JSONRPCError(RPC_INVALID_PARAMS, strprintf("range_start can only decrease; current range = [%d,%d]", existing_spk_manager->GetWalletDescriptor().range_start, existing_spk_manager->GetWalletDescriptor().range_end)); + } + } + + // Add descriptor to the wallet + auto spk_manager = pwallet->AddWalletDescriptor(w_desc, keys, label); + if (spk_manager == nullptr) { + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Could not add descriptor '%s'", descriptor)); + } + + // Set descriptor as active if necessary + if (active) { + if (!w_desc.descriptor->GetOutputType()) { + warnings.push_back("Unknown output type, cannot set descriptor to active."); + } else { + pwallet->SetActiveScriptPubKeyMan(spk_manager->GetID(), *w_desc.descriptor->GetOutputType(), internal); + } + } + + result.pushKV("success", UniValue(true)); + } catch (const UniValue& e) { + result.pushKV("success", UniValue(false)); + result.pushKV("error", e); + } catch (...) { + result.pushKV("success", UniValue(false)); + + result.pushKV("error", JSONRPCError(RPC_MISC_ERROR, "Missing required fields")); + } + if (warnings.size()) result.pushKV("warnings", warnings); + return result; +} + +UniValue importdescriptors(const JSONRPCRequest& main_request) { + // Acquire the wallet + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(main_request); + CWallet* const pwallet = wallet.get(); + if (!EnsureWalletIsAvailable(pwallet, main_request.fHelp)) { + return NullUniValue; + } + + RPCHelpMan{"importdescriptors", + "\nImport descriptors. This will trigger a rescan of the blockchain based on the earliest timestamp of all descriptors being imported. Requires a new wallet backup.\n" + "\nNote: This call can take over an hour to complete if using an early timestamp; during that time, other rpc calls\n" + "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n", + { + {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", + { + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + { + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "Descriptor to import."}, + {"active", RPCArg::Type::BOOL, /* default */ "false", "Set this descriptor to be the active descriptor for the corresponding output type/externality"}, + {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED, "If a ranged descriptor is used, this specifies the end or the range (in the form [begin,end]) to import"}, + {"next_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "If a ranged descriptor is set to active, this specifies the next index to generate addresses from"}, + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, "Time from which to start rescanning the blockchain for this descriptor, in " + UNIX_EPOCH_TIME + "\n" + " Use the string \"now\" to substitute the current synced blockchain time.\n" + " \"now\" can be specified to bypass scanning, for outputs which are known to never have been used, and\n" + " 0 can be specified to scan the entire blockchain. Blocks up to 2 hours before the earliest timestamp\n" + " of all descriptors being imported will be scanned.", + /* oneline_description */ "", {"timestamp | \"now\"", "integer / string"} + }, + {"internal", RPCArg::Type::BOOL, /* default */ "false", "Whether matching outputs should be treated as not incoming payments (e.g. change)"}, + {"label", RPCArg::Type::STR, /* default */ "''", "Label to assign to the address, only allowed with internal=false"}, + }, + }, + }, + "\"requests\""}, + }, + RPCResult{ + RPCResult::Type::ARR, "", "Response is an array with the same size as the input that has the execution result", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "success", ""}, + {RPCResult::Type::ARR, "warnings", /* optional */ true, "", + { + {RPCResult::Type::STR, "", ""}, + }}, + {RPCResult::Type::OBJ, "error", /* optional */ true, "", + { + {RPCResult::Type::ELISION, "", "JSONRPC error"}, + }}, + }}, + } + }, + RPCExamples{ + HelpExampleCli("importdescriptors", "'[{ \"desc\": \"<my descriptor>\", \"timestamp\":1455191478, \"internal\": true }, " + "{ \"desc\": \"<my desccriptor 2>\", \"label\": \"example 2\", \"timestamp\": 1455191480 }]'") + + HelpExampleCli("importdescriptors", "'[{ \"desc\": \"<my descriptor>\", \"timestamp\":1455191478, \"active\": true, \"range\": [0,100], \"label\": \"<my bech32 wallet>\" }]'") + }, + }.Check(main_request); + + // Make sure wallet is a descriptor wallet + if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "importdescriptors is not available for non-descriptor wallets"); + } + + RPCTypeCheck(main_request.params, {UniValue::VARR, UniValue::VOBJ}); + + WalletRescanReserver reserver(*pwallet); + if (!reserver.reserve()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); + } + + const UniValue& requests = main_request.params[0]; + const int64_t minimum_timestamp = 1; + int64_t now = 0; + int64_t lowest_timestamp = 0; + bool rescan = false; + UniValue response(UniValue::VARR); + { + LOCK(pwallet->cs_wallet); + EnsureWalletIsUnlocked(pwallet); + + CHECK_NONFATAL(pwallet->chain().findBlock(pwallet->GetLastBlockHash(), FoundBlock().time(lowest_timestamp).mtpTime(now))); + + // Get all timestamps and extract the lowest timestamp + for (const UniValue& request : requests.getValues()) { + // This throws an error if "timestamp" doesn't exist + const int64_t timestamp = std::max(GetImportTimestamp(request, now), minimum_timestamp); + const UniValue result = ProcessDescriptorImport(pwallet, request, timestamp); + response.push_back(result); + + if (lowest_timestamp > timestamp ) { + lowest_timestamp = timestamp; + } + + // If we know the chain tip, and at least one request was successful then allow rescan + if (!rescan && result["success"].get_bool()) { + rescan = true; + } + } + pwallet->ConnectScriptPubKeyManNotifiers(); + } + + // Rescan the blockchain using the lowest timestamp + if (rescan) { + int64_t scanned_time = pwallet->RescanFromTime(lowest_timestamp, reserver, true /* update */); + { + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(); + } + + if (pwallet->IsAbortingRescan()) { + throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); + } + + if (scanned_time > lowest_timestamp) { + std::vector<UniValue> results = response.getValues(); + response.clear(); + response.setArray(); + + // Compose the response + for (unsigned int i = 0; i < requests.size(); ++i) { + const UniValue& request = requests.getValues().at(i); + + // If the descriptor timestamp is within the successfully scanned + // range, or if the import result already has an error set, let + // the result stand unmodified. Otherwise replace the result + // with an error message. + if (scanned_time <= GetImportTimestamp(request, now) || results.at(i).exists("error")) { + response.push_back(results.at(i)); + } else { + UniValue result = UniValue(UniValue::VOBJ); + result.pushKV("success", UniValue(false)); + result.pushKV( + "error", + JSONRPCError( + RPC_MISC_ERROR, + strprintf("Rescan failed for descriptor with timestamp %d. There was an error reading a " + "block from time %d, which is after or within %d seconds of key creation, and " + "could contain transactions pertaining to the desc. As a result, transactions " + "and coins using this desc may not appear in the wallet. This error could be " + "caused by pruning or data corruption (see bitcoind log for details) and could " + "be dealt with by downloading and rescanning the relevant blocks (see -reindex " + "and -rescan options).", + GetImportTimestamp(request, now), scanned_time - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW))); + response.push_back(std::move(result)); + } + } + } + } + + return response; +} diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 46efb18793..e666d55e11 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -23,6 +23,7 @@ #include <util/moneystr.h> #include <util/string.h> #include <util/system.h> +#include <util/translation.h> #include <util/url.h> #include <util/vector.h> #include <wallet/coincontrol.h> @@ -133,7 +134,7 @@ LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_cr return *spk_man; } -static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx, UniValue& entry) +static void WalletTxToJSON(interfaces::Chain& chain, const CWalletTx& wtx, UniValue& entry) { int confirms = wtx.GetDepthInMainChain(); entry.pushKV("confirmations", confirms); @@ -148,7 +149,7 @@ static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& lo CHECK_NONFATAL(chain.findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(block_time))); entry.pushKV("blocktime", block_time); } else { - entry.pushKV("trusted", wtx.IsTrusted(locked_chain)); + entry.pushKV("trusted", wtx.IsTrusted()); } uint256 hash = wtx.GetHash(); entry.pushKV("txid", hash.GetHex()); @@ -322,7 +323,7 @@ static UniValue setlabel(const JSONRPCRequest& request) } -static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet * const pwallet, const CTxDestination &address, CAmount nValue, bool fSubtractFeeFromAmount, const CCoinControl& coin_control, mapValue_t mapValue) +static CTransactionRef SendMoney(CWallet* const pwallet, const CTxDestination& address, CAmount nValue, bool fSubtractFeeFromAmount, const CCoinControl& coin_control, mapValue_t mapValue) { CAmount curBalance = pwallet->GetBalance(0, coin_control.m_avoid_address_reuse).m_mine_trusted; @@ -338,16 +339,16 @@ static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet // Create and send the transaction CAmount nFeeRequired = 0; - std::string strError; + bilingual_str error; std::vector<CRecipient> vecSend; int nChangePosRet = -1; CRecipient recipient = {scriptPubKey, nValue, fSubtractFeeFromAmount}; vecSend.push_back(recipient); CTransactionRef tx; - if (!pwallet->CreateTransaction(locked_chain, vecSend, tx, nFeeRequired, nChangePosRet, strError, coin_control)) { + if (!pwallet->CreateTransaction(vecSend, tx, nFeeRequired, nChangePosRet, error, coin_control)) { if (!fSubtractFeeFromAmount && nValue + nFeeRequired > curBalance) - strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); - throw JSONRPCError(RPC_WALLET_ERROR, strError); + error = strprintf(Untranslated("Error: This transaction requires a transaction fee of at least %s"), FormatMoney(nFeeRequired)); + throw JSONRPCError(RPC_WALLET_ERROR, error.original); } pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */); return tx; @@ -399,7 +400,6 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); CTxDestination dest = DecodeDestination(request.params[0].get_str()); @@ -445,7 +445,7 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); - CTransactionRef tx = SendMoney(*locked_chain, pwallet, dest, nAmount, fSubtractFeeFromAmount, coin_control, std::move(mapValue)); + CTransactionRef tx = SendMoney(pwallet, dest, nAmount, fSubtractFeeFromAmount, coin_control, std::move(mapValue)); return tx->GetHash().GetHex(); } @@ -487,11 +487,10 @@ static UniValue listaddressgroupings(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); UniValue jsonGroupings(UniValue::VARR); - std::map<CTxDestination, CAmount> balances = pwallet->GetAddressBalances(*locked_chain); + std::map<CTxDestination, CAmount> balances = pwallet->GetAddressBalances(); for (const std::set<CTxDestination>& grouping : pwallet->GetAddressGroupings()) { UniValue jsonGrouping(UniValue::VARR); for (const CTxDestination& address : grouping) @@ -543,7 +542,6 @@ static UniValue signmessage(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -572,7 +570,7 @@ static UniValue signmessage(const JSONRPCRequest& request) return signature; } -static CAmount GetReceived(interfaces::Chain::Lock& locked_chain, const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { std::set<CTxDestination> address_set; @@ -602,7 +600,7 @@ static CAmount GetReceived(interfaces::Chain::Lock& locked_chain, const CWallet& CAmount amount = 0; for (const std::pair<const uint256, CWalletTx>& wtx_pair : wallet.mapWallet) { const CWalletTx& wtx = wtx_pair.second; - if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx) || wtx.GetDepthInMainChain() < min_depth) { + if (wtx.IsCoinBase() || !wallet.chain().checkFinalTx(*wtx.tx) || wtx.GetDepthInMainChain() < min_depth) { continue; } @@ -652,10 +650,9 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ false)); + return ValueFromAmount(GetReceived(*pwallet, request.params, /* by_label */ false)); } @@ -693,10 +690,9 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ true)); + return ValueFromAmount(GetReceived(*pwallet, request.params, /* by_label */ true)); } @@ -736,7 +732,6 @@ static UniValue getbalance(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); const UniValue& dummy_value = request.params[0]; @@ -778,7 +773,6 @@ static UniValue getunconfirmedbalance(const JSONRPCRequest &request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); return ValueFromAmount(pwallet->GetBalance().m_mine_untrusted_pending); @@ -841,7 +835,6 @@ static UniValue sendmany(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (!request.params[0].isNull() && !request.params[0].get_str().empty()) { @@ -911,11 +904,11 @@ static UniValue sendmany(const JSONRPCRequest& request) // Send CAmount nFeeRequired = 0; int nChangePosRet = -1; - std::string strFailReason; + bilingual_str error; CTransactionRef tx; - bool fCreated = pwallet->CreateTransaction(*locked_chain, vecSend, tx, nFeeRequired, nChangePosRet, strFailReason, coin_control); + bool fCreated = pwallet->CreateTransaction(vecSend, tx, nFeeRequired, nChangePosRet, error, coin_control); if (!fCreated) - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, error.original); pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */); return tx->GetHash().GetHex(); } @@ -963,7 +956,6 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); - auto locked_chain = pwallet->chain().lock(); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); std::string label; @@ -1016,7 +1008,7 @@ struct tallyitem } }; -static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, const CWallet* const pwallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +static UniValue ListReceived(const CWallet* const pwallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { // Minimum confirmations int nMinDepth = 1; @@ -1049,7 +1041,7 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, const CWalle for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx)) { + if (wtx.IsCoinBase() || !pwallet->chain().checkFinalTx(*wtx.tx)) { continue; } @@ -1209,10 +1201,9 @@ static UniValue listreceivedbyaddress(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - return ListReceived(*locked_chain, pwallet, request.params, false); + return ListReceived(pwallet, request.params, false); } static UniValue listreceivedbylabel(const JSONRPCRequest& request) @@ -1254,10 +1245,9 @@ static UniValue listreceivedbylabel(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - return ListReceived(*locked_chain, pwallet, request.params, true); + return ListReceived(pwallet, request.params, true); } static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) @@ -1278,7 +1268,7 @@ static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) * @param filter_ismine The "is mine" filter flags. * @param filter_label Optional label string to filter incoming transactions. */ -static void ListTransactions(interfaces::Chain::Lock& locked_chain, const CWallet* const pwallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +static void ListTransactions(const CWallet* const pwallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { CAmount nFee; std::list<COutputEntry> listReceived; @@ -1307,7 +1297,7 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, const CWalle entry.pushKV("vout", s.vout); entry.pushKV("fee", ValueFromAmount(-nFee)); if (fLong) - WalletTxToJSON(pwallet->chain(), locked_chain, wtx, entry); + WalletTxToJSON(pwallet->chain(), wtx, entry); entry.pushKV("abandoned", wtx.isAbandoned()); ret.push_back(entry); } @@ -1349,7 +1339,7 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, const CWalle } entry.pushKV("vout", r.vout); if (fLong) - WalletTxToJSON(pwallet->chain(), locked_chain, wtx, entry); + WalletTxToJSON(pwallet->chain(), wtx, entry); ret.push_back(entry); } } @@ -1464,7 +1454,6 @@ UniValue listtransactions(const JSONRPCRequest& request) UniValue ret(UniValue::VARR); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); const CWallet::TxItems & txOrdered = pwallet->wtxOrdered; @@ -1473,7 +1462,7 @@ UniValue listtransactions(const JSONRPCRequest& request) for (CWallet::TxItems::const_reverse_iterator it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) { CWalletTx *const pwtx = (*it).second; - ListTransactions(*locked_chain, pwallet, *pwtx, 0, true, ret, filter, filter_label); + ListTransactions(pwallet, *pwtx, 0, true, ret, filter, filter_label); if ((int)ret.size() >= (nCount+nFrom)) break; } } @@ -1557,7 +1546,6 @@ static UniValue listsinceblock(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); // The way the 'height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. @@ -1598,7 +1586,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) CWalletTx tx = pairWtx.second; if (depth == -1 || abs(tx.GetDepthInMainChain()) < depth) { - ListTransactions(*locked_chain, pwallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); + ListTransactions(pwallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); } } @@ -1615,7 +1603,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) if (it != pwallet->mapWallet.end()) { // We want all transactions regardless of confirmation count to appear here, // even negative confirmation ones, hence the big negative. - ListTransactions(*locked_chain, pwallet, it->second, -100000000, true, removed, filter, nullptr /* filter_label */); + ListTransactions(pwallet, it->second, -100000000, true, removed, filter, nullptr /* filter_label */); } } blockId = block.hashPrevBlock; @@ -1700,7 +1688,6 @@ static UniValue gettransaction(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); uint256 hash(ParseHashV(request.params[0], "txid")); @@ -1729,10 +1716,10 @@ static UniValue gettransaction(const JSONRPCRequest& request) if (wtx.IsFromMe(filter)) entry.pushKV("fee", ValueFromAmount(nFee)); - WalletTxToJSON(pwallet->chain(), *locked_chain, wtx, entry); + WalletTxToJSON(pwallet->chain(), wtx, entry); UniValue details(UniValue::VARR); - ListTransactions(*locked_chain, pwallet, wtx, 0, false, details, filter, nullptr /* filter_label */); + ListTransactions(pwallet, wtx, 0, false, details, filter, nullptr /* filter_label */); entry.pushKV("details", details); std::string strHex = EncodeHexTx(*wtx.tx, pwallet->chain().rpcSerializationFlags()); @@ -1776,7 +1763,6 @@ static UniValue abandontransaction(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); uint256 hash(ParseHashV(request.params[0], "txid")); @@ -1817,7 +1803,6 @@ static UniValue backupwallet(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); std::string strDest = request.params[0].get_str(); @@ -1851,11 +1836,10 @@ static UniValue keypoolrefill(const JSONRPCRequest& request) }, }.Check(request); - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (pwallet->IsLegacy() && pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); } - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); // 0 is interpreted by TopUpKeyPool() as the default keypool size given by -keypool @@ -1909,7 +1893,6 @@ static UniValue walletpassphrase(const JSONRPCRequest& request) int64_t nSleepTime; { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (!pwallet->IsCrypted()) { @@ -1991,7 +1974,6 @@ static UniValue walletpassphrasechange(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (!pwallet->IsCrypted()) { @@ -2047,7 +2029,6 @@ static UniValue walletlock(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (!pwallet->IsCrypted()) { @@ -2094,7 +2075,6 @@ static UniValue encryptwallet(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { @@ -2173,7 +2153,6 @@ static UniValue lockunspent(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); RPCTypeCheckArgument(request.params[0], UniValue::VBOOL); @@ -2286,7 +2265,6 @@ static UniValue listlockunspent(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); std::vector<COutPoint> vOutpts; @@ -2329,7 +2307,6 @@ static UniValue settxfee(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); CAmount nAmount = AmountFromValue(request.params[0]); @@ -2388,7 +2365,6 @@ static UniValue getbalances(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - auto locked_chain = wallet.chain().lock(); LOCK(wallet.cs_wallet); const auto bal = wallet.GetBalance(); @@ -2439,7 +2415,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) {RPCResult::Type::STR_AMOUNT, "unconfirmed_balance", "DEPRECATED. Identical to getbalances().mine.untrusted_pending"}, {RPCResult::Type::STR_AMOUNT, "immature_balance", "DEPRECATED. Identical to getbalances().mine.immature"}, {RPCResult::Type::NUM, "txcount", "the total number of transactions in the wallet"}, - {RPCResult::Type::NUM_TIME, "keypoololdest", "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool"}, + {RPCResult::Type::NUM_TIME, "keypoololdest", "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool. Legacy wallets only."}, {RPCResult::Type::NUM, "keypoolsize", "how many new keys are pre-generated (only counts external keys)"}, {RPCResult::Type::NUM, "keypoolsize_hd_internal", "how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)"}, {RPCResult::Type::NUM_TIME, "unlocked_until", "the " + UNIX_EPOCH_TIME + " until which the wallet is unlocked for transfers, or 0 if the wallet is locked"}, @@ -2452,6 +2428,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) {RPCResult::Type::NUM, "duration", "elapsed seconds since scan start"}, {RPCResult::Type::NUM, "progress", "scanning progress percentage [0.0, 1.0]"}, }}, + {RPCResult::Type::BOOL, "descriptors", "whether this wallet uses descriptors for scriptPubKey management"}, }}, }, RPCExamples{ @@ -2464,20 +2441,22 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); UniValue obj(UniValue::VOBJ); size_t kpExternalSize = pwallet->KeypoolCountExternalKeys(); const auto bal = pwallet->GetBalance(); + int64_t kp_oldest = pwallet->GetOldestKeyPoolTime(); obj.pushKV("walletname", pwallet->GetName()); obj.pushKV("walletversion", pwallet->GetVersion()); obj.pushKV("balance", ValueFromAmount(bal.m_mine_trusted)); obj.pushKV("unconfirmed_balance", ValueFromAmount(bal.m_mine_untrusted_pending)); obj.pushKV("immature_balance", ValueFromAmount(bal.m_mine_immature)); obj.pushKV("txcount", (int)pwallet->mapWallet.size()); - obj.pushKV("keypoololdest", pwallet->GetOldestKeyPoolTime()); + if (kp_oldest > 0) { + obj.pushKV("keypoololdest", kp_oldest); + } obj.pushKV("keypoolsize", (int64_t)kpExternalSize); LegacyScriptPubKeyMan* spk_man = pwallet->GetLegacyScriptPubKeyMan(); @@ -2505,6 +2484,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) } else { obj.pushKV("scanning", false); } + obj.pushKV("descriptors", pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); return obj; } @@ -2610,14 +2590,14 @@ static UniValue loadwallet(const JSONRPCRequest& request) } } - std::string error; - std::vector<std::string> warning; - std::shared_ptr<CWallet> const wallet = LoadWallet(*g_rpc_chain, location, error, warning); - if (!wallet) throw JSONRPCError(RPC_WALLET_ERROR, error); + bilingual_str error; + std::vector<bilingual_str> warnings; + std::shared_ptr<CWallet> const wallet = LoadWallet(*g_rpc_chain, location, error, warnings); + if (!wallet) throw JSONRPCError(RPC_WALLET_ERROR, error.original); UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); - obj.pushKV("warning", Join(warning, "\n")); + obj.pushKV("warning", Join(warnings, "\n", OpOriginal)); return obj; } @@ -2701,6 +2681,7 @@ static UniValue createwallet(const JSONRPCRequest& request) {"blank", RPCArg::Type::BOOL, /* default */ "false", "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Encrypt the wallet with this passphrase."}, {"avoid_reuse", RPCArg::Type::BOOL, /* default */ "false", "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, + {"descriptors", RPCArg::Type::BOOL, /* default */ "false", "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -2725,27 +2706,30 @@ static UniValue createwallet(const JSONRPCRequest& request) } SecureString passphrase; passphrase.reserve(100); - std::vector<std::string> warnings; + std::vector<bilingual_str> warnings; if (!request.params[3].isNull()) { passphrase = request.params[3].get_str().c_str(); if (passphrase.empty()) { // Empty string means unencrypted - warnings.emplace_back("Empty string given as passphrase, wallet will not be encrypted."); + warnings.emplace_back(Untranslated("Empty string given as passphrase, wallet will not be encrypted.")); } } if (!request.params[4].isNull() && request.params[4].get_bool()) { flags |= WALLET_FLAG_AVOID_REUSE; } + if (!request.params[5].isNull() && request.params[5].get_bool()) { + flags |= WALLET_FLAG_DESCRIPTORS; + } - std::string error; + bilingual_str error; std::shared_ptr<CWallet> wallet; WalletCreationStatus status = CreateWallet(*g_rpc_chain, passphrase, flags, request.params[0].get_str(), error, warnings, wallet); switch (status) { case WalletCreationStatus::CREATION_FAILED: - throw JSONRPCError(RPC_WALLET_ERROR, error); + throw JSONRPCError(RPC_WALLET_ERROR, error.original); case WalletCreationStatus::ENCRYPTION_FAILED: - throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, error); + throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, error.original); case WalletCreationStatus::SUCCESS: break; // no default case, so the compiler can warn about missing cases @@ -2753,7 +2737,7 @@ static UniValue createwallet(const JSONRPCRequest& request) UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); - obj.pushKV("warning", Join(warnings, "\n")); + obj.pushKV("warning", Join(warnings, "\n", OpOriginal)); return obj; } @@ -2931,9 +2915,8 @@ static UniValue listunspent(const JSONRPCRequest& request) cctl.m_avoid_address_reuse = false; cctl.m_min_depth = nMinDepth; cctl.m_max_depth = nMaxDepth; - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->AvailableCoins(*locked_chain, vecOutputs, !include_unsafe, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount); + pwallet->AvailableCoins(vecOutputs, !include_unsafe, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount); } LOCK(pwallet->cs_wallet); @@ -3126,10 +3109,10 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f setSubtractFeeFromOutputs.insert(pos); } - std::string strFailReason; + bilingual_str error; - if (!pwallet->FundTransaction(tx, fee_out, change_position, strFailReason, lockUnspents, setSubtractFeeFromOutputs, coinControl)) { - throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); + if (!pwallet->FundTransaction(tx, fee_out, change_position, error, lockUnspents, setSubtractFeeFromOutputs, coinControl)) { + throw JSONRPCError(RPC_WALLET_ERROR, error.original); } } @@ -3303,7 +3286,6 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) } // Sign the transaction - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -3432,12 +3414,11 @@ static UniValue bumpfee(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); - std::vector<std::string> errors; + std::vector<bilingual_str> errors; CAmount old_fee; CAmount new_fee; CMutableTransaction mtx; @@ -3447,19 +3428,19 @@ static UniValue bumpfee(const JSONRPCRequest& request) if (res != feebumper::Result::OK) { switch(res) { case feebumper::Result::INVALID_ADDRESS_OR_KEY: - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errors[0]); + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errors[0].original); break; case feebumper::Result::INVALID_REQUEST: - throw JSONRPCError(RPC_INVALID_REQUEST, errors[0]); + throw JSONRPCError(RPC_INVALID_REQUEST, errors[0].original); break; case feebumper::Result::INVALID_PARAMETER: - throw JSONRPCError(RPC_INVALID_PARAMETER, errors[0]); + throw JSONRPCError(RPC_INVALID_PARAMETER, errors[0].original); break; case feebumper::Result::WALLET_ERROR: - throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); + throw JSONRPCError(RPC_WALLET_ERROR, errors[0].original); break; default: - throw JSONRPCError(RPC_MISC_ERROR, errors[0]); + throw JSONRPCError(RPC_MISC_ERROR, errors[0].original); break; } } @@ -3475,7 +3456,7 @@ static UniValue bumpfee(const JSONRPCRequest& request) uint256 txid; if (feebumper::CommitTransaction(*pwallet, hash, std::move(mtx), errors, txid) != feebumper::Result::OK) { - throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); + throw JSONRPCError(RPC_WALLET_ERROR, errors[0].original); } result.pushKV("txid", txid.GetHex()); @@ -3493,8 +3474,8 @@ static UniValue bumpfee(const JSONRPCRequest& request) result.pushKV("origfee", ValueFromAmount(old_fee)); result.pushKV("fee", ValueFromAmount(new_fee)); UniValue result_errors(UniValue::VARR); - for (const std::string& error : errors) { - result_errors.push_back(error); + for (const bilingual_str& error : errors) { + result_errors.push_back(error.original); } result.pushKV("errors", result_errors); @@ -3539,7 +3520,6 @@ UniValue rescanblockchain(const JSONRPCRequest& request) Optional<int> stop_height; uint256 start_block; { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); int tip_height = pwallet->GetLastBlockHeight(); @@ -3554,8 +3534,7 @@ UniValue rescanblockchain(const JSONRPCRequest& request) stop_height = request.params[1].get_int(); if (*stop_height < 0 || *stop_height > tip_height) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stop_height"); - } - else if (*stop_height < start_height) { + } else if (*stop_height < start_height) { throw JSONRPCError(RPC_INVALID_PARAMETER, "stop_height must be greater than start_height"); } } @@ -3815,7 +3794,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) ScriptPubKeyMan* spk_man = pwallet->GetScriptPubKeyMan(scriptPubKey); if (spk_man) { - if (const CKeyMetadata* meta = spk_man->GetMetadata(dest)) { + if (const std::unique_ptr<CKeyMetadata> meta = spk_man->GetMetadata(dest)) { ret.pushKV("timestamp", meta->nCreateTime); if (meta->has_key_origin) { ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); @@ -4002,7 +3981,6 @@ UniValue sethdseed(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed to a wallet with private keys disabled"); } - auto locked_chain = pwallet->chain().lock(); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); // Do not do anything to non-HD wallets @@ -4262,12 +4240,12 @@ static UniValue upgradewallet(const JSONRPCRequest& request) version = request.params[0].get_int(); } - std::string error; - std::vector<std::string> warnings; + bilingual_str error; + std::vector<bilingual_str> warnings; if (!pwallet->UpgradeWallet(version, error, warnings)) { - throw JSONRPCError(RPC_WALLET_ERROR, error); + throw JSONRPCError(RPC_WALLET_ERROR, error.original); } - return error; + return error.original; } UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp @@ -4280,6 +4258,7 @@ UniValue importwallet(const JSONRPCRequest& request); UniValue importprunedfunds(const JSONRPCRequest& request); UniValue removeprunedfunds(const JSONRPCRequest& request); UniValue importmulti(const JSONRPCRequest& request); +UniValue importdescriptors(const JSONRPCRequest& request); void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers) { @@ -4293,7 +4272,7 @@ static const CRPCCommand commands[] = { "wallet", "addmultisigaddress", &addmultisigaddress, {"nrequired","keys","label","address_type"} }, { "wallet", "backupwallet", &backupwallet, {"destination"} }, { "wallet", "bumpfee", &bumpfee, {"txid", "options"} }, - { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse"} }, + { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse", "descriptors"} }, { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, { "wallet", "dumpwallet", &dumpwallet, {"filename"} }, { "wallet", "encryptwallet", &encryptwallet, {"passphrase"} }, @@ -4309,6 +4288,7 @@ static const CRPCCommand commands[] = { "wallet", "getbalances", &getbalances, {} }, { "wallet", "getwalletinfo", &getwalletinfo, {} }, { "wallet", "importaddress", &importaddress, {"address","label","rescan","p2sh"} }, + { "wallet", "importdescriptors", &importdescriptors, {"requests"} }, { "wallet", "importmulti", &importmulti, {"requests","options"} }, { "wallet", "importprivkey", &importprivkey, {"privkey","label","rescan"} }, { "wallet", "importprunedfunds", &importprunedfunds, {"rawtransaction","txoutproof"} }, diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index ec75f06e5e..e4be5045e1 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -377,10 +377,9 @@ bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) const return keypool_has_keys; } -bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) +bool LegacyScriptPubKeyMan::Upgrade(int prev_version, bilingual_str& error) { LOCK(cs_KeyStore); - error = ""; bool hd_upgrade = false; bool split_upgrade = false; if (m_storage.CanSupportFeature(FEATURE_HD) && !IsHDEnabled()) { @@ -405,7 +404,7 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) // Regenerate the keypool if upgraded to HD if (hd_upgrade) { if (!TopUp()) { - error = _("Unable to generate keys").translated; + error = _("Unable to generate keys"); return false; } } @@ -568,7 +567,7 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb return TransactionError::OK; } -const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const +std::unique_ptr<CKeyMetadata> LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const { LOCK(cs_KeyStore); @@ -576,14 +575,14 @@ const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& des if (!key_id.IsNull()) { auto it = mapKeyMetadata.find(key_id); if (it != mapKeyMetadata.end()) { - return &it->second; + return MakeUnique<CKeyMetadata>(it->second); } } CScript scriptPubKey = GetScriptForDestination(dest); auto it = m_script_metadata.find(CScriptID(scriptPubKey)); if (it != m_script_metadata.end()) { - return &it->second; + return MakeUnique<CKeyMetadata>(it->second); } return nullptr; @@ -1497,3 +1496,668 @@ std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const } return set_address; } + +void LegacyScriptPubKeyMan::SetType(OutputType type, bool internal) {} + +bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) +{ + // Returns true if this descriptor supports getting new addresses. Conditions where we may be unable to fetch them (e.g. locked) are caught later + if (!CanGetAddresses(m_internal)) { + error = "No addresses available"; + return false; + } + { + LOCK(cs_desc_man); + assert(m_wallet_descriptor.descriptor->IsSingleType()); // This is a combo descriptor which should not be an active descriptor + if (type != m_address_type) { + throw std::runtime_error(std::string(__func__) + ": Types are inconsistent"); + } + + TopUp(); + + // Get the scriptPubKey from the descriptor + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + if (m_wallet_descriptor.range_end <= m_max_cached_index && !TopUp(1)) { + // We can't generate anymore keys + error = "Error: Keypool ran out, please call keypoolrefill first"; + return false; + } + if (!m_wallet_descriptor.descriptor->ExpandFromCache(m_wallet_descriptor.next_index, m_wallet_descriptor.cache, scripts_temp, out_keys)) { + // We can't generate anymore keys + error = "Error: Keypool ran out, please call keypoolrefill first"; + return false; + } + + Optional<OutputType> out_script_type = m_wallet_descriptor.descriptor->GetOutputType(); + if (out_script_type && out_script_type == type) { + ExtractDestination(scripts_temp[0], dest); + } else { + throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); + } + m_wallet_descriptor.next_index++; + WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); + return true; + } +} + +isminetype DescriptorScriptPubKeyMan::IsMine(const CScript& script) const +{ + LOCK(cs_desc_man); + if (m_map_script_pub_keys.count(script) > 0) { + return ISMINE_SPENDABLE; + } + return ISMINE_NO; +} + +bool DescriptorScriptPubKeyMan::CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys) +{ + LOCK(cs_desc_man); + if (!m_map_keys.empty()) { + return false; + } + + bool keyPass = m_map_crypted_keys.empty(); // Always pass when there are no encrypted keys + bool keyFail = false; + for (const auto& mi : m_map_crypted_keys) { + const CPubKey &pubkey = mi.second.first; + const std::vector<unsigned char> &crypted_secret = mi.second.second; + CKey key; + if (!DecryptKey(master_key, crypted_secret, pubkey, key)) { + keyFail = true; + break; + } + keyPass = true; + if (m_decryption_thoroughly_checked) + break; + } + if (keyPass && keyFail) { + LogPrintf("The wallet is probably corrupted: Some keys decrypt but not all.\n"); + throw std::runtime_error("Error unlocking wallet: some keys decrypt but not all. Your wallet file may be corrupt."); + } + if (keyFail || (!keyPass && !accept_no_keys)) { + return false; + } + m_decryption_thoroughly_checked = true; + return true; +} + +bool DescriptorScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) +{ + LOCK(cs_desc_man); + if (!m_map_crypted_keys.empty()) { + return false; + } + + for (const KeyMap::value_type& key_in : m_map_keys) + { + const CKey &key = key_in.second; + CPubKey pubkey = key.GetPubKey(); + CKeyingMaterial secret(key.begin(), key.end()); + std::vector<unsigned char> crypted_secret; + if (!EncryptSecret(master_key, secret, pubkey.GetHash(), crypted_secret)) { + return false; + } + m_map_crypted_keys[pubkey.GetID()] = make_pair(pubkey, crypted_secret); + batch->WriteCryptedDescriptorKey(GetID(), pubkey, crypted_secret); + } + m_map_keys.clear(); + return true; +} + +bool DescriptorScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) +{ + LOCK(cs_desc_man); + std::string error; + bool result = GetNewDestination(type, address, error); + index = m_wallet_descriptor.next_index - 1; + return result; +} + +void DescriptorScriptPubKeyMan::ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) +{ + LOCK(cs_desc_man); + // Only return when the index was the most recent + if (m_wallet_descriptor.next_index - 1 == index) { + m_wallet_descriptor.next_index--; + } + WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); + NotifyCanGetAddressesChanged(); +} + +std::map<CKeyID, CKey> DescriptorScriptPubKeyMan::GetKeys() const +{ + AssertLockHeld(cs_desc_man); + if (m_storage.HasEncryptionKeys() && !m_storage.IsLocked()) { + KeyMap keys; + for (auto key_pair : m_map_crypted_keys) { + const CPubKey& pubkey = key_pair.second.first; + const std::vector<unsigned char>& crypted_secret = key_pair.second.second; + CKey key; + DecryptKey(m_storage.GetEncryptionKey(), crypted_secret, pubkey, key); + keys[pubkey.GetID()] = key; + } + return keys; + } + return m_map_keys; +} + +bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) +{ + LOCK(cs_desc_man); + unsigned int target_size; + if (size > 0) { + target_size = size; + } else { + target_size = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); + } + + // Calculate the new range_end + int32_t new_range_end = std::max(m_wallet_descriptor.next_index + (int32_t)target_size, m_wallet_descriptor.range_end); + + // If the descriptor is not ranged, we actually just want to fill the first cache item + if (!m_wallet_descriptor.descriptor->IsRange()) { + new_range_end = 1; + m_wallet_descriptor.range_end = 1; + m_wallet_descriptor.range_start = 0; + } + + FlatSigningProvider provider; + provider.keys = GetKeys(); + + WalletBatch batch(m_storage.GetDatabase()); + uint256 id = GetID(); + for (int32_t i = m_max_cached_index + 1; i < new_range_end; ++i) { + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + DescriptorCache temp_cache; + // Maybe we have a cached xpub and we can expand from the cache first + if (!m_wallet_descriptor.descriptor->ExpandFromCache(i, m_wallet_descriptor.cache, scripts_temp, out_keys)) { + if (!m_wallet_descriptor.descriptor->Expand(i, provider, scripts_temp, out_keys, &temp_cache)) return false; + } + // Add all of the scriptPubKeys to the scriptPubKey set + for (const CScript& script : scripts_temp) { + m_map_script_pub_keys[script] = i; + } + for (const auto& pk_pair : out_keys.pubkeys) { + const CPubKey& pubkey = pk_pair.second; + if (m_map_pubkeys.count(pubkey) != 0) { + // We don't need to give an error here. + // It doesn't matter which of many valid indexes the pubkey has, we just need an index where we can derive it and it's private key + continue; + } + m_map_pubkeys[pubkey] = i; + } + // Write the cache + for (const auto& parent_xpub_pair : temp_cache.GetCachedParentExtPubKeys()) { + CExtPubKey xpub; + if (m_wallet_descriptor.cache.GetCachedParentExtPubKey(parent_xpub_pair.first, xpub)) { + if (xpub != parent_xpub_pair.second) { + throw std::runtime_error(std::string(__func__) + ": New cached parent xpub does not match already cached parent xpub"); + } + continue; + } + if (!batch.WriteDescriptorParentCache(parent_xpub_pair.second, id, parent_xpub_pair.first)) { + throw std::runtime_error(std::string(__func__) + ": writing cache item failed"); + } + m_wallet_descriptor.cache.CacheParentExtPubKey(parent_xpub_pair.first, parent_xpub_pair.second); + } + for (const auto& derived_xpub_map_pair : temp_cache.GetCachedDerivedExtPubKeys()) { + for (const auto& derived_xpub_pair : derived_xpub_map_pair.second) { + CExtPubKey xpub; + if (m_wallet_descriptor.cache.GetCachedDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, xpub)) { + if (xpub != derived_xpub_pair.second) { + throw std::runtime_error(std::string(__func__) + ": New cached derived xpub does not match already cached derived xpub"); + } + continue; + } + if (!batch.WriteDescriptorDerivedCache(derived_xpub_pair.second, id, derived_xpub_map_pair.first, derived_xpub_pair.first)) { + throw std::runtime_error(std::string(__func__) + ": writing cache item failed"); + } + m_wallet_descriptor.cache.CacheDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, derived_xpub_pair.second); + } + } + m_max_cached_index++; + } + m_wallet_descriptor.range_end = new_range_end; + batch.WriteDescriptor(GetID(), m_wallet_descriptor); + + // By this point, the cache size should be the size of the entire range + assert(m_wallet_descriptor.range_end - 1 == m_max_cached_index); + + NotifyCanGetAddressesChanged(); + return true; +} + +void DescriptorScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) +{ + LOCK(cs_desc_man); + if (IsMine(script)) { + int32_t index = m_map_script_pub_keys[script]; + if (index >= m_wallet_descriptor.next_index) { + WalletLogPrintf("%s: Detected a used keypool item at index %d, mark all keypool items up to this item as used\n", __func__, index); + m_wallet_descriptor.next_index = index + 1; + } + if (!TopUp()) { + WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); + } + } +} + +void DescriptorScriptPubKeyMan::AddDescriptorKey(const CKey& key, const CPubKey &pubkey) +{ + LOCK(cs_desc_man); + WalletBatch batch(m_storage.GetDatabase()); + if (!AddDescriptorKeyWithDB(batch, key, pubkey)) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor private key failed"); + } +} + +bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const CKey& key, const CPubKey &pubkey) +{ + AssertLockHeld(cs_desc_man); + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + + if (m_storage.HasEncryptionKeys()) { + if (m_storage.IsLocked()) { + return false; + } + + std::vector<unsigned char> crypted_secret; + CKeyingMaterial secret(key.begin(), key.end()); + if (!EncryptSecret(m_storage.GetEncryptionKey(), secret, pubkey.GetHash(), crypted_secret)) { + return false; + } + + m_map_crypted_keys[pubkey.GetID()] = make_pair(pubkey, crypted_secret); + return batch.WriteCryptedDescriptorKey(GetID(), pubkey, crypted_secret); + } else { + m_map_keys[pubkey.GetID()] = key; + return batch.WriteDescriptorKey(GetID(), pubkey, key.GetPrivKey()); + } +} + +bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key) +{ + LOCK(cs_desc_man); + assert(m_storage.IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + + // Ignore when there is already a descriptor + if (m_wallet_descriptor.descriptor) { + return false; + } + + int64_t creation_time = GetTime(); + + std::string xpub = EncodeExtPubKey(master_key.Neuter()); + + // Build descriptor string + std::string desc_prefix; + std::string desc_suffix = "/*)"; + switch (m_address_type) { + case OutputType::LEGACY: { + desc_prefix = "pkh(" + xpub + "/44'"; + break; + } + case OutputType::P2SH_SEGWIT: { + desc_prefix = "sh(wpkh(" + xpub + "/49'"; + desc_suffix += ")"; + break; + } + case OutputType::BECH32: { + desc_prefix = "wpkh(" + xpub + "/84'"; + break; + } + default: assert(false); + } + + // Mainnet derives at 0', testnet and regtest derive at 1' + if (Params().IsTestChain()) { + desc_prefix += "/1'"; + } else { + desc_prefix += "/0'"; + } + + std::string internal_path = m_internal ? "/1" : "/0"; + std::string desc_str = desc_prefix + "/0'" + internal_path + desc_suffix; + + // Make the descriptor + FlatSigningProvider keys; + std::string error; + std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, error, false); + WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); + m_wallet_descriptor = w_desc; + + // Store the master private key, and descriptor + WalletBatch batch(m_storage.GetDatabase()); + if (!AddDescriptorKeyWithDB(batch, master_key.key, master_key.key.GetPubKey())) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor master private key failed"); + } + if (!batch.WriteDescriptor(GetID(), m_wallet_descriptor)) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor failed"); + } + + // TopUp + TopUp(); + + m_storage.UnsetBlankWalletFlag(batch); + return true; +} + +bool DescriptorScriptPubKeyMan::IsHDEnabled() const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.descriptor->IsRange(); +} + +bool DescriptorScriptPubKeyMan::CanGetAddresses(bool internal) const +{ + // We can only give out addresses from descriptors that are single type (not combo), ranged, + // and either have cached keys or can generate more keys (ignoring encryption) + LOCK(cs_desc_man); + return m_wallet_descriptor.descriptor->IsSingleType() && + m_wallet_descriptor.descriptor->IsRange() && + (HavePrivateKeys() || m_wallet_descriptor.next_index < m_wallet_descriptor.range_end); +} + +bool DescriptorScriptPubKeyMan::HavePrivateKeys() const +{ + LOCK(cs_desc_man); + return m_map_keys.size() > 0 || m_map_crypted_keys.size() > 0; +} + +int64_t DescriptorScriptPubKeyMan::GetOldestKeyPoolTime() const +{ + // This is only used for getwalletinfo output and isn't relevant to descriptor wallets. + // The magic number 0 indicates that it shouldn't be displayed so that's what we return. + return 0; +} + +size_t DescriptorScriptPubKeyMan::KeypoolCountExternalKeys() const +{ + if (m_internal) { + return 0; + } + return GetKeyPoolSize(); +} + +unsigned int DescriptorScriptPubKeyMan::GetKeyPoolSize() const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.range_end - m_wallet_descriptor.next_index; +} + +int64_t DescriptorScriptPubKeyMan::GetTimeFirstKey() const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.creation_time; +} + +std::unique_ptr<FlatSigningProvider> DescriptorScriptPubKeyMan::GetSigningProvider(const CScript& script, bool include_private) const +{ + LOCK(cs_desc_man); + + // Find the index of the script + auto it = m_map_script_pub_keys.find(script); + if (it == m_map_script_pub_keys.end()) { + return nullptr; + } + int32_t index = it->second; + + return GetSigningProvider(index, include_private); +} + +std::unique_ptr<FlatSigningProvider> DescriptorScriptPubKeyMan::GetSigningProvider(const CPubKey& pubkey) const +{ + LOCK(cs_desc_man); + + // Find index of the pubkey + auto it = m_map_pubkeys.find(pubkey); + if (it == m_map_pubkeys.end()) { + return nullptr; + } + int32_t index = it->second; + + // Always try to get the signing provider with private keys. This function should only be called during signing anyways + return GetSigningProvider(index, true); +} + +std::unique_ptr<FlatSigningProvider> DescriptorScriptPubKeyMan::GetSigningProvider(int32_t index, bool include_private) const +{ + AssertLockHeld(cs_desc_man); + // Get the scripts, keys, and key origins for this script + std::unique_ptr<FlatSigningProvider> out_keys = MakeUnique<FlatSigningProvider>(); + std::vector<CScript> scripts_temp; + if (!m_wallet_descriptor.descriptor->ExpandFromCache(index, m_wallet_descriptor.cache, scripts_temp, *out_keys)) return nullptr; + + if (HavePrivateKeys() && include_private) { + FlatSigningProvider master_provider; + master_provider.keys = GetKeys(); + m_wallet_descriptor.descriptor->ExpandPrivate(index, master_provider, *out_keys); + } + + return out_keys; +} + +std::unique_ptr<SigningProvider> DescriptorScriptPubKeyMan::GetSolvingProvider(const CScript& script) const +{ + return GetSigningProvider(script, false); +} + +bool DescriptorScriptPubKeyMan::CanProvide(const CScript& script, SignatureData& sigdata) +{ + return IsMine(script); +} + +bool DescriptorScriptPubKeyMan::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const +{ + std::unique_ptr<FlatSigningProvider> keys = MakeUnique<FlatSigningProvider>(); + for (const auto& coin_pair : coins) { + std::unique_ptr<FlatSigningProvider> coin_keys = GetSigningProvider(coin_pair.second.out.scriptPubKey, true); + if (!coin_keys) { + continue; + } + *keys = Merge(*keys, *coin_keys); + } + + return ::SignTransaction(tx, keys.get(), coins, sighash, input_errors); +} + +SigningResult DescriptorScriptPubKeyMan::SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const +{ + std::unique_ptr<FlatSigningProvider> keys = GetSigningProvider(GetScriptForDestination(pkhash), true); + if (!keys) { + return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; + } + + CKeyID key_id(pkhash); + CKey key; + if (!keys->GetKey(key_id, key)) { + return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; + } + + if (!MessageSign(key, message, str_sig)) { + return SigningResult::SIGNING_FAILED; + } + return SigningResult::OK; +} + +TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, int sighash_type, bool sign, bool bip32derivs) const +{ + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + const CTxIn& txin = psbtx.tx->vin[i]; + PSBTInput& input = psbtx.inputs.at(i); + + if (PSBTInputSigned(input)) { + continue; + } + + // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. + if (!input.IsSane()) { + return TransactionError::INVALID_PSBT; + } + + // Get the Sighash type + if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { + return TransactionError::SIGHASH_MISMATCH; + } + + // Get the scriptPubKey to know which SigningProvider to use + CScript script; + if (!input.witness_utxo.IsNull()) { + script = input.witness_utxo.scriptPubKey; + } else if (input.non_witness_utxo) { + if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { + return TransactionError::MISSING_INPUTS; + } + script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; + } else { + // There's no UTXO so we can just skip this now + continue; + } + SignatureData sigdata; + input.FillSignatureData(sigdata); + + std::unique_ptr<FlatSigningProvider> keys = MakeUnique<FlatSigningProvider>(); + std::unique_ptr<FlatSigningProvider> script_keys = GetSigningProvider(script, sign); + if (script_keys) { + *keys = Merge(*keys, *script_keys); + } else { + // Maybe there are pubkeys listed that we can sign for + script_keys = MakeUnique<FlatSigningProvider>(); + for (const auto& pk_pair : input.hd_keypaths) { + const CPubKey& pubkey = pk_pair.first; + std::unique_ptr<FlatSigningProvider> pk_keys = GetSigningProvider(pubkey); + if (pk_keys) { + *keys = Merge(*keys, *pk_keys); + } + } + } + + SignPSBTInput(HidingSigningProvider(keys.get(), !sign, !bip32derivs), psbtx, i, sighash_type); + } + + // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change + for (unsigned int i = 0; i < psbtx.tx->vout.size(); ++i) { + std::unique_ptr<SigningProvider> keys = GetSolvingProvider(psbtx.tx->vout.at(i).scriptPubKey); + if (!keys) { + continue; + } + UpdatePSBTOutput(HidingSigningProvider(keys.get(), true, !bip32derivs), psbtx, i); + } + + return TransactionError::OK; +} + +std::unique_ptr<CKeyMetadata> DescriptorScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const +{ + std::unique_ptr<SigningProvider> provider = GetSigningProvider(GetScriptForDestination(dest)); + if (provider) { + KeyOriginInfo orig; + CKeyID key_id = GetKeyForDestination(*provider, dest); + if (provider->GetKeyOrigin(key_id, orig)) { + LOCK(cs_desc_man); + std::unique_ptr<CKeyMetadata> meta = MakeUnique<CKeyMetadata>(); + meta->key_origin = orig; + meta->has_key_origin = true; + meta->nCreateTime = m_wallet_descriptor.creation_time; + return meta; + } + } + return nullptr; +} + +uint256 DescriptorScriptPubKeyMan::GetID() const +{ + LOCK(cs_desc_man); + std::string desc_str = m_wallet_descriptor.descriptor->ToString(); + uint256 id; + CSHA256().Write((unsigned char*)desc_str.data(), desc_str.size()).Finalize(id.begin()); + return id; +} + +void DescriptorScriptPubKeyMan::SetType(OutputType type, bool internal) +{ + this->m_address_type = type; + this->m_internal = internal; +} + +void DescriptorScriptPubKeyMan::SetCache(const DescriptorCache& cache) +{ + LOCK(cs_desc_man); + m_wallet_descriptor.cache = cache; + for (int32_t i = m_wallet_descriptor.range_start; i < m_wallet_descriptor.range_end; ++i) { + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + if (!m_wallet_descriptor.descriptor->ExpandFromCache(i, m_wallet_descriptor.cache, scripts_temp, out_keys)) { + throw std::runtime_error("Error: Unable to expand wallet descriptor from cache"); + } + // Add all of the scriptPubKeys to the scriptPubKey set + for (const CScript& script : scripts_temp) { + if (m_map_script_pub_keys.count(script) != 0) { + throw std::runtime_error(strprintf("Error: Already loaded script at index %d as being at index %d", i, m_map_script_pub_keys[script])); + } + m_map_script_pub_keys[script] = i; + } + for (const auto& pk_pair : out_keys.pubkeys) { + const CPubKey& pubkey = pk_pair.second; + if (m_map_pubkeys.count(pubkey) != 0) { + // We don't need to give an error here. + // It doesn't matter which of many valid indexes the pubkey has, we just need an index where we can derive it and it's private key + continue; + } + m_map_pubkeys[pubkey] = i; + } + m_max_cached_index++; + } +} + +bool DescriptorScriptPubKeyMan::AddKey(const CKeyID& key_id, const CKey& key) +{ + LOCK(cs_desc_man); + m_map_keys[key_id] = key; + return true; +} + +bool DescriptorScriptPubKeyMan::AddCryptedKey(const CKeyID& key_id, const CPubKey& pubkey, const std::vector<unsigned char>& crypted_key) +{ + LOCK(cs_desc_man); + if (!m_map_keys.empty()) { + return false; + } + + m_map_crypted_keys[key_id] = make_pair(pubkey, crypted_key); + return true; +} + +bool DescriptorScriptPubKeyMan::HasWalletDescriptor(const WalletDescriptor& desc) const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.descriptor != nullptr && desc.descriptor != nullptr && m_wallet_descriptor.descriptor->ToString() == desc.descriptor->ToString(); +} + +void DescriptorScriptPubKeyMan::WriteDescriptor() +{ + LOCK(cs_desc_man); + WalletBatch batch(m_storage.GetDatabase()); + if (!batch.WriteDescriptor(GetID(), m_wallet_descriptor)) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor failed"); + } +} + +const WalletDescriptor DescriptorScriptPubKeyMan::GetWalletDescriptor() const +{ + return m_wallet_descriptor; +} + +const std::vector<CScript> DescriptorScriptPubKeyMan::GetScriptPubKeys() const +{ + LOCK(cs_desc_man); + std::vector<CScript> script_pub_keys; + script_pub_keys.reserve(m_map_script_pub_keys.size()); + + for (auto const& script_pub_key: m_map_script_pub_keys) { + script_pub_keys.push_back(script_pub_key.first); + } + return script_pub_keys; +} diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 8512eadf31..4c002edf2d 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -6,6 +6,7 @@ #define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H #include <psbt.h> +#include <script/descriptor.h> #include <script/signingprovider.h> #include <script/standard.h> #include <util/error.h> @@ -18,6 +19,7 @@ #include <boost/signals2/signal.hpp> enum class OutputType; +struct bilingual_str; // Wallet storage things that ScriptPubKeyMans need in order to be able to store things to the wallet database. // It provides access to things that are part of the entire wallet and not specific to a ScriptPubKeyMan such as @@ -190,7 +192,7 @@ public: virtual bool CanGetAddresses(bool internal = false) const { return false; } /** Upgrades the wallet to the specified version */ - virtual bool Upgrade(int prev_version, std::string& error) { return false; } + virtual bool Upgrade(int prev_version, bilingual_str& error) { return false; } virtual bool HavePrivateKeys() const { return false; } @@ -204,7 +206,7 @@ public: virtual int64_t GetTimeFirstKey() const { return 0; } - virtual const CKeyMetadata* GetMetadata(const CTxDestination& dest) const { return nullptr; } + virtual std::unique_ptr<CKeyMetadata> GetMetadata(const CTxDestination& dest) const { return nullptr; } virtual std::unique_ptr<SigningProvider> GetSolvingProvider(const CScript& script) const { return nullptr; } @@ -222,6 +224,8 @@ public: virtual uint256 GetID() const { return uint256(); } + virtual void SetType(OutputType type, bool internal) {} + /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ template<typename... Params> void WalletLogPrintf(std::string fmt, Params... parameters) const { @@ -340,7 +344,7 @@ public: bool SetupGeneration(bool force = false) override; - bool Upgrade(int prev_version, std::string& error) override; + bool Upgrade(int prev_version, bilingual_str& error) override; bool HavePrivateKeys() const override; @@ -352,7 +356,7 @@ public: int64_t GetTimeFirstKey() const override; - const CKeyMetadata* GetMetadata(const CTxDestination& dest) const override; + std::unique_ptr<CKeyMetadata> GetMetadata(const CTxDestination& dest) const override; bool CanGetAddresses(bool internal = false) const override; @@ -366,6 +370,8 @@ public: uint256 GetID() const override; + void SetType(OutputType type, bool internal) override; + // Map from Key ID to key metadata. std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_KeyStore); @@ -477,4 +483,111 @@ public: bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override { return m_spk_man.GetKeyOrigin(keyid, info); } }; +class DescriptorScriptPubKeyMan : public ScriptPubKeyMan +{ +private: + WalletDescriptor m_wallet_descriptor GUARDED_BY(cs_desc_man); + + using ScriptPubKeyMap = std::map<CScript, int32_t>; // Map of scripts to descriptor range index + using PubKeyMap = std::map<CPubKey, int32_t>; // Map of pubkeys involved in scripts to descriptor range index + using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; + using KeyMap = std::map<CKeyID, CKey>; + + ScriptPubKeyMap m_map_script_pub_keys GUARDED_BY(cs_desc_man); + PubKeyMap m_map_pubkeys GUARDED_BY(cs_desc_man); + int32_t m_max_cached_index = -1; + + OutputType m_address_type; + bool m_internal = false; + + KeyMap m_map_keys GUARDED_BY(cs_desc_man); + CryptedKeyMap m_map_crypted_keys GUARDED_BY(cs_desc_man); + + bool SetCrypted(); + + //! keeps track of whether Unlock has run a thorough check before + bool m_decryption_thoroughly_checked = false; + + bool AddDescriptorKeyWithDB(WalletBatch& batch, const CKey& key, const CPubKey &pubkey); + + KeyMap GetKeys() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + + // Fetch the SigningProvider for the given script and optionally include private keys + std::unique_ptr<FlatSigningProvider> GetSigningProvider(const CScript& script, bool include_private = false) const; + // Fetch the SigningProvider for the given pubkey and always include private keys. This should only be called by signing code. + std::unique_ptr<FlatSigningProvider> GetSigningProvider(const CPubKey& pubkey) const; + // Fetch the SigningProvider for a given index and optionally include private keys. Called by the above functions. + std::unique_ptr<FlatSigningProvider> GetSigningProvider(int32_t index, bool include_private = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + +public: + DescriptorScriptPubKeyMan(WalletStorage& storage, WalletDescriptor& descriptor) + : ScriptPubKeyMan(storage), + m_wallet_descriptor(descriptor) + {} + DescriptorScriptPubKeyMan(WalletStorage& storage, OutputType address_type, bool internal) + : ScriptPubKeyMan(storage), + m_address_type(address_type), m_internal(internal) + {} + + mutable RecursiveMutex cs_desc_man; + + bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; + isminetype IsMine(const CScript& script) const override; + + bool CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys = false) override; + bool Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) override; + + bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) override; + void ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) override; + + // Tops up the descriptor cache and m_map_script_pub_keys. The cache is stored in the wallet file + // and is used to expand the descriptor in GetNewDestination. DescriptorScriptPubKeyMan relies + // more on ephemeral data than LegacyScriptPubKeyMan. For wallets using unhardened derivation + // (with or without private keys), the "keypool" is a single xpub. + bool TopUp(unsigned int size = 0) override; + + void MarkUnusedAddresses(const CScript& script) override; + + bool IsHDEnabled() const override; + + //! Setup descriptors based on the given CExtkey + bool SetupDescriptorGeneration(const CExtKey& master_key); + + bool HavePrivateKeys() const override; + + int64_t GetOldestKeyPoolTime() const override; + size_t KeypoolCountExternalKeys() const override; + unsigned int GetKeyPoolSize() const override; + + int64_t GetTimeFirstKey() const override; + + std::unique_ptr<CKeyMetadata> GetMetadata(const CTxDestination& dest) const override; + + bool CanGetAddresses(bool internal = false) const override; + + std::unique_ptr<SigningProvider> GetSolvingProvider(const CScript& script) const override; + + bool CanProvide(const CScript& script, SignatureData& sigdata) override; + + bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const override; + SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; + TransactionError FillPSBT(PartiallySignedTransaction& psbt, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false) const override; + + uint256 GetID() const override; + + void SetType(OutputType type, bool internal) override; + + void SetCache(const DescriptorCache& cache); + + bool AddKey(const CKeyID& key_id, const CKey& key); + bool AddCryptedKey(const CKeyID& key_id, const CPubKey& pubkey, const std::vector<unsigned char>& crypted_key); + + bool HasWalletDescriptor(const WalletDescriptor& desc) const; + void AddDescriptorKey(const CKey& key, const CPubKey &pubkey); + void WriteDescriptor(); + + const WalletDescriptor GetWalletDescriptor() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + const std::vector<CScript> GetScriptPubKeys() const; +}; + #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 42ae738207..0826b88f0a 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -4,6 +4,7 @@ #include <wallet/wallet.h> +#include <future> #include <memory> #include <stdint.h> #include <vector> @@ -12,7 +13,9 @@ #include <node/context.h> #include <policy/policy.h> #include <rpc/server.h> +#include <test/util/logging.h> #include <test/util/setup_common.h> +#include <util/translation.h> #include <validation.h> #include <wallet/coincontrol.h> #include <wallet/test/wallet_test_fixture.h> @@ -26,6 +29,36 @@ extern UniValue importwallet(const JSONRPCRequest& request); BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) +static std::shared_ptr<CWallet> TestLoadWallet(interfaces::Chain& chain) +{ + bilingual_str error; + std::vector<bilingual_str> warnings; + auto wallet = CWallet::CreateWalletFromFile(chain, WalletLocation(""), error, warnings); + wallet->postInitProcess(); + return wallet; +} + +static void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet) +{ + SyncWithValidationInterfaceQueue(); + wallet->m_chain_notifications_handler.reset(); + UnloadWallet(std::move(wallet)); +} + +static CMutableTransaction TestSimpleSpend(const CTransaction& from, uint32_t index, const CKey& key, const CScript& pubkey) +{ + CMutableTransaction mtx; + mtx.vout.push_back({from.vout[index].nValue - DEFAULT_TRANSACTION_MAXFEE, pubkey}); + mtx.vin.push_back({CTxIn{from.GetHash(), index}}); + FillableSigningProvider keystore; + keystore.AddKey(key); + std::map<COutPoint, Coin> coins; + coins[mtx.vin[0].prevout].out = from.vout[index]; + std::map<int, std::string> input_errors; + BOOST_CHECK(SignTransaction(mtx, &keystore, coins, SIGHASH_ALL, input_errors)); + return mtx; +} + static void AddKey(CWallet& wallet, const CKey& key) { auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); @@ -43,8 +76,6 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) NodeContext node; auto chain = interfaces::MakeChain(node); - auto locked_chain = chain->lock(); - LockAssertion lock(::cs_main); // Verify ScanForWalletTransactions fails to read an unknown start block. { @@ -84,7 +115,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) } // Prune the older block file. - PruneOneBlockFile(oldTip->GetBlockPos().nFile); + { + LOCK(cs_main); + PruneOneBlockFile(oldTip->GetBlockPos().nFile); + } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); // Verify ScanForWalletTransactions only picks transactions in the new block @@ -107,7 +141,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) } // Prune the remaining block file. - PruneOneBlockFile(newTip->GetBlockPos().nFile); + { + LOCK(cs_main); + PruneOneBlockFile(newTip->GetBlockPos().nFile); + } UnlinkPrunedFiles({newTip->GetBlockPos().nFile}); // Verify ScanForWalletTransactions scans no blocks. @@ -139,11 +176,12 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) NodeContext node; auto chain = interfaces::MakeChain(node); - auto locked_chain = chain->lock(); - LockAssertion lock(::cs_main); // Prune the older block file. - PruneOneBlockFile(oldTip->GetBlockPos().nFile); + { + LOCK(cs_main); + PruneOneBlockFile(oldTip->GetBlockPos().nFile); + } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); // Verify importmulti RPC returns failure for a key whose creation time is @@ -209,8 +247,6 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) NodeContext node; auto chain = interfaces::MakeChain(node); - auto locked_chain = chain->lock(); - LockAssertion lock(::cs_main); std::string backup_file = (GetDataDir() / "wallet.backup").string(); @@ -276,8 +312,6 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); CWalletTx wtx(&wallet, m_coinbase_txns.back()); - auto locked_chain = chain->lock(); - LockAssertion lock(::cs_main); LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -302,8 +336,6 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 SetMockTime(mockTime); CBlockIndex* block = nullptr; if (blockTime > 0) { - auto locked_chain = wallet.chain().lock(); - LockAssertion lock(::cs_main); auto inserted = ::BlockIndex().emplace(GetRandHash(), new CBlockIndex); assert(inserted.second); const uint256& hash = inserted.first->first; @@ -313,7 +345,6 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 } CWalletTx wtx(&wallet, MakeTransactionRef(tx)); - LOCK(cs_main); LOCK(wallet.cs_wallet); // If transaction is already in map, to avoid inconsistencies, unconfirmation // is needed before confirm again with different block. @@ -460,7 +491,7 @@ public: CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); { - LOCK2(::cs_main, wallet->cs_wallet); + LOCK2(wallet->cs_wallet, ::cs_main); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } bool firstRun; @@ -485,11 +516,10 @@ public: CTransactionRef tx; CAmount fee; int changePos = -1; - std::string error; + bilingual_str error; CCoinControl dummy; { - auto locked_chain = m_chain->lock(); - BOOST_CHECK(wallet->CreateTransaction(*locked_chain, {recipient}, tx, fee, changePos, error, dummy)); + BOOST_CHECK(wallet->CreateTransaction({recipient}, tx, fee, changePos, error, dummy)); } wallet->CommitTransaction(tx, {}, {}); CMutableTransaction blocktx; @@ -499,7 +529,6 @@ public: } CreateAndProcessBlock({CMutableTransaction(blocktx)}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - LOCK(cs_main); LOCK(wallet->cs_wallet); wallet->SetLastBlockProcessed(wallet->GetLastBlockHeight() + 1, ::ChainActive().Tip()->GetBlockHash()); auto it = wallet->mapWallet.find(tx->GetHash()); @@ -522,9 +551,8 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) // address. std::map<CTxDestination, std::vector<COutput>> list; { - auto locked_chain = m_chain->lock(); LOCK(wallet->cs_wallet); - list = wallet->ListCoins(*locked_chain); + list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); @@ -539,9 +567,8 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) // pubkey. AddTx(CRecipient{GetScriptForRawPubKey({}), 1 * COIN, false /* subtract fee */}); { - auto locked_chain = m_chain->lock(); LOCK(wallet->cs_wallet); - list = wallet->ListCoins(*locked_chain); + list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); @@ -549,10 +576,9 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) // Lock both coins. Confirm number of available coins drops to 0. { - auto locked_chain = m_chain->lock(); LOCK(wallet->cs_wallet); std::vector<COutput> available; - wallet->AvailableCoins(*locked_chain, available); + wallet->AvailableCoins(available); BOOST_CHECK_EQUAL(available.size(), 2U); } for (const auto& group : list) { @@ -562,18 +588,16 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) } } { - auto locked_chain = m_chain->lock(); LOCK(wallet->cs_wallet); std::vector<COutput> available; - wallet->AvailableCoins(*locked_chain, available); + wallet->AvailableCoins(available); BOOST_CHECK_EQUAL(available.size(), 0U); } // Confirm ListCoins still returns same result as before, despite coins // being locked. { - auto locked_chain = m_chain->lock(); LOCK(wallet->cs_wallet); - list = wallet->ListCoins(*locked_chain); + list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); @@ -637,4 +661,137 @@ BOOST_FIXTURE_TEST_CASE(dummy_input_size_test, TestChain100Setup) BOOST_CHECK_EQUAL(CalculateNestedKeyhashInputSize(true), DUMMY_NESTED_P2WPKH_INPUT_SIZE); } +bool malformed_descriptor(std::ios_base::failure e) +{ + std::string s(e.what()); + return s.find("Missing checksum") != std::string::npos; +} + +BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) +{ + std::vector<unsigned char> malformed_record; + CVectorWriter vw(0, 0, malformed_record, 0); + vw << std::string("notadescriptor"); + vw << (uint64_t)0; + vw << (int32_t)0; + vw << (int32_t)0; + vw << (int32_t)1; + + VectorReader vr(0, 0, malformed_record, 0); + WalletDescriptor w_desc; + BOOST_CHECK_EXCEPTION(vr >> w_desc, std::ios_base::failure, malformed_descriptor); +} + +//! Test CreateWalletFromFile function and its behavior handling potential race +//! conditions if it's called the same time an incoming transaction shows up in +//! the mempool or a new block. +//! +//! It isn't possible to verify there aren't race condition in every case, so +//! this test just checks two specific cases and ensures that timing of +//! notifications in these cases doesn't prevent the wallet from detecting +//! transactions. +//! +//! In the first case, block and mempool transactions are created before the +//! wallet is loaded, but notifications about these transactions are delayed +//! until after it is loaded. The notifications are superfluous in this case, so +//! the test verifies the transactions are detected before they arrive. +//! +//! In the second case, block and mempool transactions are created after the +//! wallet rescan and notifications are immediately synced, to verify the wallet +//! must already have a handler in place for them, and there's no gap after +//! rescanning where new transactions in new blocks could be lost. +BOOST_FIXTURE_TEST_CASE(CreateWalletFromFile, TestChain100Setup) +{ + // Create new wallet with known key and unload it. + auto chain = interfaces::MakeChain(m_node); + auto wallet = TestLoadWallet(*chain); + CKey key; + key.MakeNewKey(true); + AddKey(*wallet, key); + TestUnloadWallet(std::move(wallet)); + + + // Add log hook to detect AddToWallet events from rescans, blockConnected, + // and transactionAddedToMempool notifications + int addtx_count = 0; + DebugLogHelper addtx_counter("[default wallet] AddToWallet", [&](const std::string* s) { + if (s) ++addtx_count; + return false; + }); + + + bool rescan_completed = false; + DebugLogHelper rescan_check("[default wallet] Rescan completed", [&](const std::string* s) { + if (s) rescan_completed = true; + return false; + }); + + + // Block the queue to prevent the wallet receiving blockConnected and + // transactionAddedToMempool notifications, and create block and mempool + // transactions paying to the wallet + std::promise<void> promise; + CallFunctionInValidationInterfaceQueue([&promise] { + promise.get_future().wait(); + }); + std::string error; + m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + auto block_tx = TestSimpleSpend(*m_coinbase_txns[0], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + m_coinbase_txns.push_back(CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + auto mempool_tx = TestSimpleSpend(*m_coinbase_txns[1], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + BOOST_CHECK(chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); + + + // Reload wallet and make sure new transactions are detected despite events + // being blocked + wallet = TestLoadWallet(*chain); + BOOST_CHECK(rescan_completed); + BOOST_CHECK_EQUAL(addtx_count, 2); + { + LOCK(wallet->cs_wallet); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(mempool_tx.GetHash()), 1); + } + + + // Unblock notification queue and make sure stale blockConnected and + // transactionAddedToMempool events are processed + promise.set_value(); + SyncWithValidationInterfaceQueue(); + BOOST_CHECK_EQUAL(addtx_count, 4); + + + TestUnloadWallet(std::move(wallet)); + + + // Load wallet again, this time creating new block and mempool transactions + // paying to the wallet as the wallet finishes loading and syncing the + // queue so the events have to be handled immediately. Releasing the wallet + // lock during the sync is a little artificial but is needed to avoid a + // deadlock during the sync and simulates a new block notification happening + // as soon as possible. + addtx_count = 0; + auto handler = HandleLoadWallet([&](std::unique_ptr<interfaces::Wallet> wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet->wallet()->cs_wallet) { + BOOST_CHECK(rescan_completed); + m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + block_tx = TestSimpleSpend(*m_coinbase_txns[2], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + m_coinbase_txns.push_back(CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + mempool_tx = TestSimpleSpend(*m_coinbase_txns[3], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + BOOST_CHECK(chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); + LEAVE_CRITICAL_SECTION(wallet->wallet()->cs_wallet); + SyncWithValidationInterfaceQueue(); + ENTER_CRITICAL_SECTION(wallet->wallet()->cs_wallet); + }); + wallet = TestLoadWallet(*chain); + BOOST_CHECK_EQUAL(addtx_count, 4); + { + LOCK(wallet->cs_wallet); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(mempool_tx.GetHash()), 1); + } + + + TestUnloadWallet(std::move(wallet)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 7a6deab1a8..a893548971 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -27,6 +27,7 @@ #include <util/fees.h> #include <util/moneystr.h> #include <util/rbf.h> +#include <util/string.h> #include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/fees.h> @@ -149,34 +150,34 @@ void UnloadWallet(std::shared_ptr<CWallet>&& wallet) } } -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::vector<std::string>& warnings) +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) { try { if (!CWallet::Verify(chain, location, false, error, warnings)) { - error = "Wallet file verification failed: " + error; + error = Untranslated("Wallet file verification failed.") + Untranslated(" ") + error; return nullptr; } std::shared_ptr<CWallet> wallet = CWallet::CreateWalletFromFile(chain, location, error, warnings); if (!wallet) { - error = "Wallet loading failed: " + error; + error = Untranslated("Wallet loading failed.") + Untranslated(" ") + error; return nullptr; } AddWallet(wallet); wallet->postInitProcess(); return wallet; } catch (const std::runtime_error& e) { - error = e.what(); + error = Untranslated(e.what()); return nullptr; } } -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::vector<std::string>& warnings) +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) { return LoadWallet(chain, WalletLocation(name), error, warnings); } -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result) +WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result) { // Indicate that the wallet is actually supposed to be blank and not just blank to make it encrypted bool create_blank = (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET); @@ -189,49 +190,53 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& // Check the wallet file location WalletLocation location(name); if (location.Exists()) { - error = "Wallet " + location.GetName() + " already exists."; + error = strprintf(Untranslated("Wallet %s already exists."), location.GetName()); return WalletCreationStatus::CREATION_FAILED; } // Wallet::Verify will check if we're trying to create a wallet with a duplicate name. if (!CWallet::Verify(chain, location, false, error, warnings)) { - error = "Wallet file verification failed: " + error; + error = Untranslated("Wallet file verification failed.") + Untranslated(" ") + error; return WalletCreationStatus::CREATION_FAILED; } // Do not allow a passphrase when private keys are disabled if (!passphrase.empty() && (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - error = "Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled."; + error = Untranslated("Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled."); return WalletCreationStatus::CREATION_FAILED; } // Make the wallet std::shared_ptr<CWallet> wallet = CWallet::CreateWalletFromFile(chain, location, error, warnings, wallet_creation_flags); if (!wallet) { - error = "Wallet creation failed: " + error; + error = Untranslated("Wallet creation failed.") + Untranslated(" ") + error; return WalletCreationStatus::CREATION_FAILED; } // Encrypt the wallet if (!passphrase.empty() && !(wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { if (!wallet->EncryptWallet(passphrase)) { - error = "Error: Wallet created but failed to encrypt."; + error = Untranslated("Error: Wallet created but failed to encrypt."); return WalletCreationStatus::ENCRYPTION_FAILED; } if (!create_blank) { // Unlock the wallet if (!wallet->Unlock(passphrase)) { - error = "Error: Wallet was encrypted but could not be unlocked"; + error = Untranslated("Error: Wallet was encrypted but could not be unlocked"); return WalletCreationStatus::ENCRYPTION_FAILED; } // Set a seed for the wallet { LOCK(wallet->cs_wallet); - for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { - if (!spk_man->SetupGeneration()) { - error = "Unable to generate initial keys"; - return WalletCreationStatus::CREATION_FAILED; + if (wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + wallet->SetupDescriptorScriptPubKeyMans(); + } else { + for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { + if (!spk_man->SetupGeneration()) { + error = Untranslated("Unable to generate initial keys"); + return WalletCreationStatus::CREATION_FAILED; + } } } } @@ -588,8 +593,11 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) Lock(); Unlock(strWalletPassphrase); - // if we are using HD, replace the HD seed with a new one - if (auto spk_man = GetLegacyScriptPubKeyMan()) { + // If we are using descriptors, make new descriptors with a new seed + if (IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)) { + SetupDescriptorScriptPubKeyMans(); + } else if (auto spk_man = GetLegacyScriptPubKeyMan()) { + // if we are using HD, replace the HD seed with a new one if (spk_man->IsHDEnabled()) { if (!spk_man->SetupGeneration(true)) { return false; @@ -747,22 +755,29 @@ bool CWallet::IsSpentKey(const uint256& hash, unsigned int n) const const CWalletTx* srctx = GetWalletTx(hash); if (srctx) { assert(srctx->tx->vout.size() > n); - LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); - // When descriptor wallets arrive, these additional checks are - // likely superfluous and can be optimized out - assert(spk_man != nullptr); - for (const auto& keyid : GetAffectedKeys(srctx->tx->vout[n].scriptPubKey, *spk_man)) { - WitnessV0KeyHash wpkh_dest(keyid); - if (GetDestData(wpkh_dest, "used", nullptr)) { - return true; - } - ScriptHash sh_wpkh_dest(GetScriptForDestination(wpkh_dest)); - if (GetDestData(sh_wpkh_dest, "used", nullptr)) { - return true; - } - PKHash pkh_dest(keyid); - if (GetDestData(pkh_dest, "used", nullptr)) { - return true; + CTxDestination dest; + if (!ExtractDestination(srctx->tx->vout[n].scriptPubKey, dest)) { + return false; + } + if (GetDestData(dest, "used", nullptr)) { + return true; + } + if (IsLegacy()) { + LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); + assert(spk_man != nullptr); + for (const auto& keyid : GetAffectedKeys(srctx->tx->vout[n].scriptPubKey, *spk_man)) { + WitnessV0KeyHash wpkh_dest(keyid); + if (GetDestData(wpkh_dest, "used", nullptr)) { + return true; + } + ScriptHash sh_wpkh_dest(GetScriptForDestination(wpkh_dest)); + if (GetDestData(sh_wpkh_dest, "used", nullptr)) { + return true; + } + PKHash pkh_dest(keyid); + if (GetDestData(pkh_dest, "used", nullptr)) { + return true; + } } } } @@ -871,10 +886,9 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) void CWallet::LoadToWallet(CWalletTx& wtxIn) { - // If wallet doesn't have a chain (e.g bitcoin-wallet), lock can't be taken. - auto locked_chain = LockChain(); - if (locked_chain) { - Optional<int> block_height = locked_chain->getBlockHeight(wtxIn.m_confirm.hashBlock); + // If wallet doesn't have a chain (e.g wallet-tool), don't bother to update txn. + if (HaveChain()) { + Optional<int> block_height = chain().getBlockHeight(wtxIn.m_confirm.hashBlock); if (block_height) { // Update cached block height variable since it not stored in the // serialized transaction. @@ -961,7 +975,6 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Co bool CWallet::TransactionCanBeAbandoned(const uint256& hashTx) const { - auto locked_chain = chain().lock(); LOCK(cs_wallet); const CWalletTx* wtx = GetWalletTx(hashTx); return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain() == 0 && !wtx->InMempool(); @@ -979,7 +992,6 @@ void CWallet::MarkInputsDirty(const CTransactionRef& tx) bool CWallet::AbandonTransaction(const uint256& hashTx) { - auto locked_chain = chain().lock(); // Temporary. Removed in upcoming lock cleanup LOCK(cs_wallet); WalletBatch batch(*database, "r+"); @@ -1034,7 +1046,6 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); int conflictconfirms = (m_last_block_processed_height - conflicting_height + 1) * -1; @@ -1097,7 +1108,6 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmatio } void CWallet::transactionAddedToMempool(const CTransactionRef& ptx) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); SyncTransaction(ptx, confirm); @@ -1119,7 +1129,6 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef &ptx) { void CWallet::blockConnected(const CBlock& block, int height) { const uint256& block_hash = block.GetHash(); - auto locked_chain = chain().lock(); LOCK(cs_wallet); m_last_block_processed_height = height; @@ -1133,7 +1142,6 @@ void CWallet::blockConnected(const CBlock& block, int height) void CWallet::blockDisconnected(const CBlock& block, int height) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); // At block disconnection, this will change an abandoned transaction to @@ -1331,9 +1339,10 @@ CAmount CWallet::GetChange(const CTransaction& tx) const bool CWallet::IsHDEnabled() const { + // All Active ScriptPubKeyMans must be HD for this to be true bool result = true; - for (const auto& spk_man_pair : m_spk_managers) { - result &= spk_man_pair.second->IsHDEnabled(); + for (const auto& spk_man : GetActiveScriptPubKeyMans()) { + result &= spk_man->IsHDEnabled(); } return result; } @@ -1671,7 +1680,6 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc uint256 next_block_hash; bool reorg = false; if (chain().findBlock(block_hash, FoundBlock().data(block)) && !block.IsNull()) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); next_block = chain().findNextBlock(block_hash, block_height, FoundBlock().hash(next_block_hash), &reorg); if (reorg) { @@ -1700,7 +1708,6 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } { - auto locked_chain = chain().lock(); if (!next_block || reorg) { // break successfully when rescan has reached the tip, or // previous block is no longer on the chain due to a reorg @@ -1913,16 +1920,16 @@ bool CWalletTx::InMempool() const return fInMempool; } -bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain) const +bool CWalletTx::IsTrusted() const { std::set<uint256> s; - return IsTrusted(locked_chain, s); + return IsTrusted(s); } -bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const +bool CWalletTx::IsTrusted(std::set<uint256>& trusted_parents) const { // Quick answer in most cases - if (!locked_chain.checkFinalTx(*tx)) return false; + if (!pwallet->chain().checkFinalTx(*tx)) return false; int nDepth = GetDepthInMainChain(); if (nDepth >= 1) return true; if (nDepth < 0) return false; @@ -1944,7 +1951,7 @@ bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint25 // If we've already trusted this parent, continue if (trusted_parents.count(parent->GetHash())) continue; // Recurse to check that the parent is also trusted - if (!parent->IsTrusted(locked_chain, trusted_parents)) return false; + if (!parent->IsTrusted(trusted_parents)) return false; trusted_parents.insert(parent->GetHash()); } return true; @@ -1978,7 +1985,8 @@ void CWallet::ResendWalletTransactions() // that these are our transactions. if (GetTime() < nNextResend || !fBroadcastTransactions) return; bool fFirst = (nNextResend == 0); - nNextResend = GetTime() + GetRand(30 * 60); + // resend 12-36 hours from now, ~1 day on average. + nNextResend = GetTime() + (12 * 60 * 60) + GetRand(24 * 60 * 60); if (fFirst) return; // Only do it if there's been a new block since last time @@ -1987,8 +1995,7 @@ void CWallet::ResendWalletTransactions() int submitted_tx_count = 0; - { // locked_chain and cs_wallet scope - auto locked_chain = chain().lock(); + { // cs_wallet scope LOCK(cs_wallet); // Relay transactions @@ -2001,7 +2008,7 @@ void CWallet::ResendWalletTransactions() std::string unused_err_string; if (wtx.SubmitMemoryPoolAndRelay(unused_err_string, true)) ++submitted_tx_count; } - } // locked_chain and cs_wallet + } // cs_wallet if (submitted_tx_count > 0) { WalletLogPrintf("%s: resubmit %u unconfirmed transactions\n", __func__, submitted_tx_count); @@ -2029,13 +2036,12 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons Balance ret; isminefilter reuse_filter = avoid_reuse ? ISMINE_NO : ISMINE_USED; { - auto locked_chain = chain().lock(); LOCK(cs_wallet); std::set<uint256> trusted_parents; for (const auto& entry : mapWallet) { const CWalletTx& wtx = entry.second; - const bool is_trusted{wtx.IsTrusted(*locked_chain, trusted_parents)}; + const bool is_trusted{wtx.IsTrusted(trusted_parents)}; const int tx_depth{wtx.GetDepthInMainChain()}; const CAmount tx_credit_mine{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; @@ -2056,12 +2062,11 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const { - auto locked_chain = chain().lock(); LOCK(cs_wallet); CAmount balance = 0; std::vector<COutput> vCoins; - AvailableCoins(*locked_chain, vCoins, true, coinControl); + AvailableCoins(vCoins, true, coinControl); for (const COutput& out : vCoins) { if (out.fSpendable) { balance += out.tx->tx->vout[out.i].nValue; @@ -2070,7 +2075,7 @@ CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const return balance; } -void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<COutput>& vCoins, bool fOnlySafe, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount) const +void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount) const { AssertLockHeld(cs_wallet); @@ -2088,7 +2093,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< const uint256& wtxid = entry.first; const CWalletTx& wtx = entry.second; - if (!locked_chain.checkFinalTx(*wtx.tx)) { + if (!chain().checkFinalTx(*wtx.tx)) { continue; } @@ -2104,7 +2109,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = wtx.IsTrusted(locked_chain, trusted_parents); + bool safeTx = wtx.IsTrusted(trusted_parents); // We should not consider coins from transactions that are replacing // other transactions. @@ -2192,14 +2197,14 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< } } -std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins(interfaces::Chain::Lock& locked_chain) const +std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins() const { AssertLockHeld(cs_wallet); std::map<CTxDestination, std::vector<COutput>> result; std::vector<COutput> availableCoins; - AvailableCoins(locked_chain, availableCoins); + AvailableCoins(availableCoins); for (const COutput& coin : availableCoins) { CTxDestination address; @@ -2427,11 +2432,17 @@ bool CWallet::SignTransaction(CMutableTransaction& tx) const bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const { - // Sign the tx with ScriptPubKeyMans - // Because each ScriptPubKeyMan can sign more than one input, we need to keep track of each ScriptPubKeyMan that has signed this transaction. - // Each iteration, we may sign more txins than the txin that is specified in that iteration. - // We assume that each input is signed by only one ScriptPubKeyMan. - std::set<uint256> visited_spk_mans; + // Try to sign with all ScriptPubKeyMans + for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { + // spk_man->SignTransaction will return true if the transaction is complete, + // so we can exit early and return true if that happens + if (spk_man->SignTransaction(tx, coins, sighash, input_errors)) { + return true; + } + } + + // At this point, one input was not fully signed otherwise we would have exited already + // Find that input and figure out what went wrong. for (unsigned int i = 0; i < tx.vin.size(); i++) { // Get the prevout CTxIn& txin = tx.vin[i]; @@ -2443,33 +2454,10 @@ bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, // Check if this input is complete SignatureData sigdata = DataFromTransaction(tx, i, coin->second.out); - if (sigdata.complete) { - continue; - } - - // Input needs to be signed, find the right ScriptPubKeyMan - std::set<ScriptPubKeyMan*> spk_mans = GetScriptPubKeyMans(coin->second.out.scriptPubKey, sigdata); - if (spk_mans.size() == 0) { + if (!sigdata.complete) { input_errors[i] = "Unable to sign input, missing keys"; continue; } - - for (auto& spk_man : spk_mans) { - // If we've already been signed by this spk_man, skip it - if (visited_spk_mans.count(spk_man->GetID()) > 0) { - continue; - } - - // Sign the tx. - // spk_man->SignTransaction will return true if the transaction is complete, - // so we can exit early and return true if that happens. - if (spk_man->SignTransaction(tx, coins, sighash, input_errors)) { - return true; - } - - // Add this spk_man to visited_spk_mans so we can skip it later - visited_spk_mans.insert(spk_man->GetID()); - } } return false; } @@ -2505,52 +2493,10 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp } // Fill in information from ScriptPubKeyMans - // Because each ScriptPubKeyMan may be able to fill more than one input, we need to keep track of each ScriptPubKeyMan that has filled this psbt. - // Each iteration, we may fill more inputs than the input that is specified in that iteration. - // We assume that each input is filled by only one ScriptPubKeyMan - std::set<uint256> visited_spk_mans; - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - const CTxIn& txin = psbtx.tx->vin[i]; - PSBTInput& input = psbtx.inputs.at(i); - - if (PSBTInputSigned(input)) { - continue; - } - - // Get the scriptPubKey to know which ScriptPubKeyMan to use - CScript script; - if (!input.witness_utxo.IsNull()) { - script = input.witness_utxo.scriptPubKey; - } else if (input.non_witness_utxo) { - if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { - return TransactionError::MISSING_INPUTS; - } - script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; - } else { - // There's no UTXO so we can just skip this now - continue; - } - SignatureData sigdata; - input.FillSignatureData(sigdata); - std::set<ScriptPubKeyMan*> spk_mans = GetScriptPubKeyMans(script, sigdata); - if (spk_mans.size() == 0) { - continue; - } - - for (auto& spk_man : spk_mans) { - // If we've already been signed by this spk_man, skip it - if (visited_spk_mans.count(spk_man->GetID()) > 0) { - continue; - } - - // Fill in the information from the spk_man - TransactionError res = spk_man->FillPSBT(psbtx, sighash_type, sign, bip32derivs); - if (res != TransactionError::OK) { - return res; - } - - // Add this spk_man to visited_spk_mans so we can skip it later - visited_spk_mans.insert(spk_man->GetID()); + for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { + TransactionError res = spk_man->FillPSBT(psbtx, sighash_type, sign, bip32derivs); + if (res != TransactionError::OK) { + return res; } } @@ -2575,7 +2521,7 @@ SigningResult CWallet::SignMessage(const std::string& message, const PKHash& pkh return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; } -bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl coinControl) +bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl coinControl) { std::vector<CRecipient> vecSend; @@ -2594,11 +2540,10 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC // Acquire the locks to prevent races to the new locked unspents between the // CreateTransaction call and LockCoin calls (when lockUnspents is true). - auto locked_chain = chain().lock(); LOCK(cs_wallet); CTransactionRef tx_new; - if (!CreateTransaction(*locked_chain, vecSend, tx_new, nFeeRet, nChangePosInOut, strFailReason, coinControl, false)) { + if (!CreateTransaction(vecSend, tx_new, nFeeRet, nChangePosInOut, error, coinControl, false)) { return false; } @@ -2714,8 +2659,7 @@ OutputType CWallet::TransactionChangeType(OutputType change_type, const std::vec return m_default_address_type; } -bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, - int& nChangePosInOut, std::string& strFailReason, const CCoinControl& coin_control, bool sign) +bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, bool sign) { CAmount nValue = 0; const OutputType change_type = TransactionChangeType(coin_control.m_change_type ? *coin_control.m_change_type : m_default_change_type, vecSend); @@ -2726,7 +2670,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std { if (nValue < 0 || recipient.nAmount < 0) { - strFailReason = _("Transaction amounts must not be negative").translated; + error = _("Transaction amounts must not be negative"); return false; } nValue += recipient.nAmount; @@ -2736,7 +2680,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } if (vecSend.empty()) { - strFailReason = _("Transaction must have at least one recipient").translated; + error = _("Transaction must have at least one recipient"); return false; } @@ -2746,12 +2690,11 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std int nBytes; { std::set<CInputCoin> setCoins; - auto locked_chain = chain().lock(); LOCK(cs_wallet); txNew.nLockTime = GetLocktimeForNewTransaction(chain(), GetLastBlockHash(), GetLastBlockHeight()); { std::vector<COutput> vAvailableCoins; - AvailableCoins(*locked_chain, vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); + AvailableCoins(vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); CoinSelectionParams coin_selection_params; // Parameters for coin selection, init with dummy // Create change script that will be used if we need change @@ -2774,7 +2717,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // destination in case we don't need change. CTxDestination dest; if (!reservedest.GetReservedDestination(dest, true)) { - strFailReason = _("Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.").translated; + error = _("Transaction needs a change address, but we can't generate it. Please call keypoolrefill first."); } scriptChange = GetScriptForDestination(dest); assert(!dest.empty() || scriptChange.empty()); @@ -2836,12 +2779,12 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std if (recipient.fSubtractFeeFromAmount && nFeeRet > 0) { if (txout.nValue < 0) - strFailReason = _("The transaction amount is too small to pay the fee").translated; + error = _("The transaction amount is too small to pay the fee"); else - strFailReason = _("The transaction amount is too small to send after the fee has been deducted").translated; + error = _("The transaction amount is too small to send after the fee has been deducted"); } else - strFailReason = _("Transaction amount too small").translated; + error = _("Transaction amount too small"); return false; } txNew.vout.push_back(txout); @@ -2869,7 +2812,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std continue; } else { - strFailReason = _("Insufficient funds").translated; + error = _("Insufficient funds"); return false; } } @@ -2900,7 +2843,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } else if ((unsigned int)nChangePosInOut > txNew.vout.size()) { - strFailReason = _("Change index out of range").translated; + error = _("Change index out of range"); return false; } @@ -2919,14 +2862,14 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std nBytes = CalculateMaximumSignedTxSize(CTransaction(txNew), this, coin_control.fAllowWatchOnly); if (nBytes < 0) { - strFailReason = _("Signing transaction failed").translated; + error = _("Signing transaction failed"); return false; } nFeeNeeded = GetMinimumFee(*this, nBytes, coin_control, &feeCalc); if (feeCalc.reason == FeeReason::FALLBACK && !m_allow_fallback_fee) { // eventually allow a fallback fee - strFailReason = _("Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee.").translated; + error = _("Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee."); return false; } @@ -2966,7 +2909,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // fee to pay for the new output and still meet nFeeNeeded // Or we should have just subtracted fee from recipients and // nFeeNeeded should not have changed - strFailReason = _("Transaction fee and change calculation failed").translated; + error = _("Transaction fee and change calculation failed"); return false; } @@ -3019,7 +2962,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } if (sign && !SignTransaction(txNew)) { - strFailReason = _("Signing transaction failed").translated; + error = _("Signing transaction failed"); return false; } @@ -3029,20 +2972,20 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // Limit size if (GetTransactionWeight(*tx) > MAX_STANDARD_TX_WEIGHT) { - strFailReason = _("Transaction too large").translated; + error = _("Transaction too large"); return false; } } if (nFeeRet > m_default_max_tx_fee) { - strFailReason = TransactionErrorString(TransactionError::MAX_FEE_EXCEEDED); + error = Untranslated(TransactionErrorString(TransactionError::MAX_FEE_EXCEEDED)); return false; } if (gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS)) { // Lastly, ensure this tx will pass the mempool's chain limits if (!chain().checkChainLimits(tx)) { - strFailReason = _("Transaction has too long of a mempool chain").translated; + error = _("Transaction has too long of a mempool chain"); return false; } } @@ -3064,7 +3007,6 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); CWalletTx wtxNew(this, std::move(tx)); @@ -3104,11 +3046,6 @@ void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { - // Even if we don't use this lock in this function, we want to preserve - // lock order in LoadToWallet if query of chain state is needed to know - // tx status. If lock can't be taken (e.g bitcoin-wallet), tx confirmation - // status may be not reliable. - auto locked_chain = LockChain(); LOCK(cs_wallet); fFirstRunRet = false; @@ -3278,6 +3215,8 @@ bool CWallet::GetNewDestination(const OutputType type, const std::string label, if (spk_man) { spk_man->TopUp(); result = spk_man->GetNewDestination(type, dest, error); + } else { + error = strprintf("Error: No %s addresses available.", FormatOutputType(type)); } if (result) { SetAddressBook(dest, label, "receive"); @@ -3325,7 +3264,7 @@ void CWallet::MarkDestinationsDirty(const std::set<CTxDestination>& destinations } } -std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain::Lock& locked_chain) const +std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() const { std::map<CTxDestination, CAmount> balances; @@ -3336,7 +3275,7 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: { const CWalletTx& wtx = walletEntry.second; - if (!wtx.IsTrusted(locked_chain, trusted_parents)) + if (!wtx.IsTrusted(trusted_parents)) continue; if (wtx.IsImmatureCoinBase()) @@ -3552,7 +3491,7 @@ void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const /** @} */ // end of Actions -void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<CKeyID, int64_t>& mapKeyBirth) const { +void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { AssertLockHeld(cs_wallet); mapKeyBirth.clear(); @@ -3718,7 +3657,7 @@ std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const return values; } -bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, std::string& error_string, std::vector<std::string>& warnings) +bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, bilingual_str& error_string, std::vector<bilingual_str>& warnings) { // Do some checking on wallet path. It should be either a: // @@ -3732,17 +3671,17 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b if (!(path_type == fs::file_not_found || path_type == fs::directory_file || (path_type == fs::symlink_file && fs::is_directory(wallet_path)) || (path_type == fs::regular_file && fs::path(location.GetName()).filename() == location.GetName()))) { - error_string = strprintf( + error_string = Untranslated(strprintf( "Invalid -wallet path '%s'. -wallet path should point to a directory where wallet.dat and " "database/log.?????????? files can be stored, a location where such a directory could be created, " "or (for backwards compatibility) the name of an existing data file in -walletdir (%s)", - location.GetName(), GetWalletDir()); + location.GetName(), GetWalletDir())); return false; } // Make sure that the wallet path doesn't clash with an existing wallet path if (IsWalletLoaded(wallet_path)) { - error_string = strprintf("Error loading wallet %s. Duplicate -wallet filename specified.", location.GetName()); + error_string = Untranslated(strprintf("Error loading wallet %s. Duplicate -wallet filename specified.", location.GetName())); return false; } @@ -3754,7 +3693,7 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b return false; } } catch (const fs::filesystem_error& e) { - error_string = strprintf("Error loading wallet %s. %s", location.GetName(), fsbridge::get_filesystem_error_message(e)); + error_string = Untranslated(strprintf("Error loading wallet %s. %s", location.GetName(), fsbridge::get_filesystem_error_message(e))); return false; } @@ -3762,11 +3701,6 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b // Recover readable keypairs: CWallet dummyWallet(&chain, WalletLocation(), WalletDatabase::CreateDummy()); std::string backup_filename; - // Even if we don't use this lock in this function, we want to preserve - // lock order in LoadToWallet if query of chain state is needed to know - // tx status. If lock can't be taken, tx confirmation status may be not - // reliable. - auto locked_chain = dummyWallet.LockChain(); if (!WalletBatch::Recover(wallet_path, (void *)&dummyWallet, WalletBatch::RecoverKeysOnlyFilter, backup_filename)) { return false; } @@ -3775,7 +3709,7 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b return WalletBatch::VerifyDatabaseFile(wallet_path, warnings, error_string); } -std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::vector<std::string>& warnings, uint64_t wallet_creation_flags) +std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags) { const std::string walletFile = WalletDataFilePath(location.GetPath()).string(); @@ -3788,7 +3722,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(&chain, location, WalletDatabase::Create(location.GetPath())); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DBErrors::LOAD_OK) { - error = strprintf(_("Error loading %s: Wallet corrupted").translated, walletFile); + error = strprintf(_("Error loading %s: Wallet corrupted"), walletFile); return nullptr; } } @@ -3803,26 +3737,26 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun); if (nLoadWalletRet != DBErrors::LOAD_OK) { if (nLoadWalletRet == DBErrors::CORRUPT) { - error = strprintf(_("Error loading %s: Wallet corrupted").translated, walletFile); + error = strprintf(_("Error loading %s: Wallet corrupted"), walletFile); return nullptr; } else if (nLoadWalletRet == DBErrors::NONCRITICAL_ERROR) { warnings.push_back(strprintf(_("Error reading %s! All keys read correctly, but transaction data" - " or address book entries might be missing or incorrect.").translated, + " or address book entries might be missing or incorrect."), walletFile)); } else if (nLoadWalletRet == DBErrors::TOO_NEW) { - error = strprintf(_("Error loading %s: Wallet requires newer version of %s").translated, walletFile, PACKAGE_NAME); + error = strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, PACKAGE_NAME); return nullptr; } else if (nLoadWalletRet == DBErrors::NEED_REWRITE) { - error = strprintf(_("Wallet needed to be rewritten: restart %s to complete").translated, PACKAGE_NAME); + error = strprintf(_("Wallet needed to be rewritten: restart %s to complete"), PACKAGE_NAME); return nullptr; } else { - error = strprintf(_("Error loading %s").translated, walletFile); + error = strprintf(_("Error loading %s"), walletFile); return nullptr; } } @@ -3834,53 +3768,60 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->SetWalletFlags(wallet_creation_flags, false); - // Always create LegacyScriptPubKeyMan for now - walletInstance->SetupLegacyScriptPubKeyMan(); + // Only create LegacyScriptPubKeyMan when not descriptor wallet + if (!walletInstance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + walletInstance->SetupLegacyScriptPubKeyMan(); + } if (!(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { LOCK(walletInstance->cs_wallet); - for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { - if (!spk_man->SetupGeneration()) { - error = _("Unable to generate initial keys").translated; - return nullptr; + if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + walletInstance->SetupDescriptorScriptPubKeyMans(); + // SetupDescriptorScriptPubKeyMans already calls SetupGeneration for us so we don't need to call SetupGeneration separately + } else { + // Legacy wallets need SetupGeneration here. + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { + if (!spk_man->SetupGeneration()) { + error = _("Unable to generate initial keys"); + return nullptr; + } } } } - auto locked_chain = chain.lock(); - walletInstance->chainStateFlushed(locked_chain->getTipLocator()); + walletInstance->chainStateFlushed(chain.getTipLocator()); } else if (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS) { // Make it impossible to disable private keys after creation - error = strprintf(_("Error loading %s: Private keys can only be disabled during creation").translated, walletFile); + error = strprintf(_("Error loading %s: Private keys can only be disabled during creation"), walletFile); return NULL; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { if (spk_man->HavePrivateKeys()) { - warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys").translated, walletFile)); + warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys"), walletFile)); break; } } } if (!gArgs.GetArg("-addresstype", "").empty() && !ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { - error = strprintf(_("Unknown address type '%s'").translated, gArgs.GetArg("-addresstype", "")); + error = strprintf(_("Unknown address type '%s'"), gArgs.GetArg("-addresstype", "")); return nullptr; } if (!gArgs.GetArg("-changetype", "").empty() && !ParseOutputType(gArgs.GetArg("-changetype", ""), walletInstance->m_default_change_type)) { - error = strprintf(_("Unknown change type '%s'").translated, gArgs.GetArg("-changetype", "")); + error = strprintf(_("Unknown change type '%s'"), gArgs.GetArg("-changetype", "")); return nullptr; } if (gArgs.IsArgSet("-mintxfee")) { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-mintxfee", ""), n) || 0 == n) { - error = AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", "")).translated; + error = AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", "")); return nullptr; } if (n > HIGH_TX_FEE_PER_KB) { - warnings.push_back(AmountHighWarn("-mintxfee").translated + " " + - _("This is the minimum transaction fee you pay on every transaction.").translated); + warnings.push_back(AmountHighWarn("-mintxfee") + Untranslated(" ") + + _("This is the minimum transaction fee you pay on every transaction.")); } walletInstance->m_min_fee = CFeeRate(n); } @@ -3888,12 +3829,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-fallbackfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-fallbackfee", ""), nFeePerK)) { - error = strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'").translated, gArgs.GetArg("-fallbackfee", "")); + error = strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'"), gArgs.GetArg("-fallbackfee", "")); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - warnings.push_back(AmountHighWarn("-fallbackfee").translated + " " + - _("This is the transaction fee you may pay when fee estimates are not available.").translated); + warnings.push_back(AmountHighWarn("-fallbackfee") + Untranslated(" ") + + _("This is the transaction fee you may pay when fee estimates are not available.")); } walletInstance->m_fallback_fee = CFeeRate(nFeePerK); } @@ -3903,28 +3844,28 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-discardfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-discardfee", ""), nFeePerK)) { - error = strprintf(_("Invalid amount for -discardfee=<amount>: '%s'").translated, gArgs.GetArg("-discardfee", "")); + error = strprintf(_("Invalid amount for -discardfee=<amount>: '%s'"), gArgs.GetArg("-discardfee", "")); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - warnings.push_back(AmountHighWarn("-discardfee").translated + " " + - _("This is the transaction fee you may discard if change is smaller than dust at this level").translated); + warnings.push_back(AmountHighWarn("-discardfee") + Untranslated(" ") + + _("This is the transaction fee you may discard if change is smaller than dust at this level")); } walletInstance->m_discard_rate = CFeeRate(nFeePerK); } if (gArgs.IsArgSet("-paytxfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-paytxfee", ""), nFeePerK)) { - error = AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", "")).translated; + error = AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", "")); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - warnings.push_back(AmountHighWarn("-paytxfee").translated + " " + - _("This is the transaction fee you will pay if you send a transaction.").translated); + warnings.push_back(AmountHighWarn("-paytxfee") + Untranslated(" ") + + _("This is the transaction fee you will pay if you send a transaction.")); } walletInstance->m_pay_tx_fee = CFeeRate(nFeePerK, 1000); if (walletInstance->m_pay_tx_fee < chain.relayMinFee()) { - error = strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)").translated, + error = strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)"), gArgs.GetArg("-paytxfee", ""), chain.relayMinFee().ToString()); return nullptr; } @@ -3933,23 +3874,23 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-maxtxfee")) { CAmount nMaxFee = 0; if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) { - error = AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", "")).translated; + error = AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", "")); return nullptr; } if (nMaxFee > HIGH_MAX_TX_FEE) { - warnings.push_back(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.").translated); + warnings.push_back(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); } if (CFeeRate(nMaxFee, 1000) < chain.relayMinFee()) { - error = strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)").translated, - gArgs.GetArg("-maxtxfee", ""), chain.relayMinFee().ToString()); + error = strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), + gArgs.GetArg("-maxtxfee", ""), chain.relayMinFee().ToString()); return nullptr; } walletInstance->m_default_max_tx_fee = nMaxFee; } if (chain.relayMinFee().GetFeePerK() > HIGH_TX_FEE_PER_KB) { - warnings.push_back(AmountHighWarn("-minrelaytxfee").translated + " " + - _("The wallet will avoid paying less than the minimum relay fee.").translated); + warnings.push_back(AmountHighWarn("-minrelaytxfee") + Untranslated(" ") + + _("The wallet will avoid paying less than the minimum relay fee.")); } walletInstance->m_confirm_target = gArgs.GetArg("-txconfirmtarget", DEFAULT_TX_CONFIRM_TARGET); @@ -3961,24 +3902,33 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // Try to top up keypool. No-op if the wallet is locked. walletInstance->TopUpKeyPool(); - auto locked_chain = chain.lock(); LOCK(walletInstance->cs_wallet); + // Register wallet with validationinterface. It's done before rescan to avoid + // missing block connections between end of rescan and validation subscribing. + // Because of wallet lock being hold, block connection notifications are going to + // be pending on the validation-side until lock release. It's likely to have + // block processing duplicata (if rescan block range overlaps with notification one) + // but we guarantee at least than wallet state is correct after notifications delivery. + // This is temporary until rescan and notifications delivery are unified under same + // interface. + walletInstance->m_chain_notifications_handler = walletInstance->chain().handleNotifications(walletInstance); + int rescan_height = 0; if (!gArgs.GetBoolArg("-rescan", false)) { WalletBatch batch(*walletInstance->database); CBlockLocator locator; if (batch.ReadBestBlock(locator)) { - if (const Optional<int> fork_height = locked_chain->findLocatorFork(locator)) { + if (const Optional<int> fork_height = chain.findLocatorFork(locator)) { rescan_height = *fork_height; } } } - const Optional<int> tip_height = locked_chain->getHeight(); + const Optional<int> tip_height = chain.getHeight(); if (tip_height) { - walletInstance->m_last_block_processed = locked_chain->getBlockHash(*tip_height); + walletInstance->m_last_block_processed = chain.getBlockHash(*tip_height); walletInstance->m_last_block_processed_height = *tip_height; } else { walletInstance->m_last_block_processed.SetNull(); @@ -3995,12 +3945,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // If a block is pruned after this check, we will load the wallet, // but fail the rescan with a generic error. int block_height = *tip_height; - while (block_height > 0 && locked_chain->haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { + while (block_height > 0 && chain.haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { --block_height; } if (rescan_height != block_height) { - error = _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)").translated; + error = _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); return nullptr; } } @@ -4017,19 +3967,19 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (!time_first_key || time < *time_first_key) time_first_key = time; } if (time_first_key) { - if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, nullptr)) { + if (Optional<int> first_block = chain.findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, nullptr)) { rescan_height = *first_block; } } { WalletRescanReserver reserver(*walletInstance); - if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { - error = _("Failed to rescan the wallet during initialization").translated; + if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(chain.getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { + error = _("Failed to rescan the wallet during initialization"); return nullptr; } } - walletInstance->chainStateFlushed(locked_chain->getTipLocator()); + walletInstance->chainStateFlushed(chain.getTipLocator()); walletInstance->database->IncrementUpdateCounter(); // Restore wallet transaction metadata after -zapwallettxes=1 @@ -4064,9 +4014,6 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - // Register with the validation interface. It's ok to do this after rescan since we're still holding locked_chain. - walletInstance->m_chain_notifications_handler = walletInstance->chain().handleNotifications(walletInstance); - walletInstance->SetBroadcastTransactions(gArgs.GetBoolArg("-walletbroadcast", DEFAULT_WALLETBROADCAST)); { @@ -4088,7 +4035,7 @@ const CAddressBookData* CWallet::FindAddressBookEntry(const CTxDestination& dest return &address_book_it->second; } -bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings) +bool CWallet::UpgradeWallet(int version, bilingual_str& error, std::vector<bilingual_str>& warnings) { int prev_version = GetVersion(); int nMaxVersion = version; @@ -4097,12 +4044,12 @@ bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::st WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); nMaxVersion = FEATURE_LATEST; SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately - } - else + } else { WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); + } if (nMaxVersion < GetVersion()) { - error = _("Cannot downgrade wallet").translated; + error = _("Cannot downgrade wallet"); return false; } SetMaxVersion(nMaxVersion); @@ -4112,7 +4059,7 @@ bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::st // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT int max_version = GetVersion(); if (!CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { - error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified.").translated; + error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified."); return false; } @@ -4126,7 +4073,6 @@ bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::st void CWallet::postInitProcess() { - auto locked_chain = chain().lock(); LOCK(cs_wallet); // Add wallet transactions that aren't already in a block to mempool @@ -4353,6 +4299,9 @@ std::unique_ptr<SigningProvider> CWallet::GetSolvingProvider(const CScript& scri LegacyScriptPubKeyMan* CWallet::GetLegacyScriptPubKeyMan() const { + if (IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + return nullptr; + } // Legacy wallets only have one ScriptPubKeyMan which is a LegacyScriptPubKeyMan. // Everything in m_internal_spk_managers and m_external_spk_managers point to the same legacyScriptPubKeyMan. auto it = m_internal_spk_managers.find(OutputType::LEGACY); @@ -4368,7 +4317,7 @@ LegacyScriptPubKeyMan* CWallet::GetOrCreateLegacyScriptPubKeyMan() void CWallet::SetupLegacyScriptPubKeyMan() { - if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty()) { + if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty() || IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { return; } @@ -4397,3 +4346,153 @@ void CWallet::ConnectScriptPubKeyManNotifiers() spk_man->NotifyCanGetAddressesChanged.connect(NotifyCanGetAddressesChanged); } } + +void CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc) +{ + auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc)); + m_spk_managers[id] = std::move(spk_manager); +} + +void CWallet::SetupDescriptorScriptPubKeyMans() +{ + AssertLockHeld(cs_wallet); + + // Make a seed + CKey seed_key; + seed_key.MakeNewKey(true); + CPubKey seed = seed_key.GetPubKey(); + assert(seed_key.VerifyPubKey(seed)); + + // Get the extended key + CExtKey master_key; + master_key.SetSeed(seed_key.begin(), seed_key.size()); + + for (bool internal : {false, true}) { + for (OutputType t : OUTPUT_TYPES) { + auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, t, internal)); + if (IsCrypted()) { + if (IsLocked()) { + throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); + } + if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, nullptr)) { + throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); + } + } + spk_manager->SetupDescriptorGeneration(master_key); + uint256 id = spk_manager->GetID(); + m_spk_managers[id] = std::move(spk_manager); + SetActiveScriptPubKeyMan(id, t, internal); + } + } +} + +void CWallet::SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal, bool memonly) +{ + WalletLogPrintf("Setting spkMan to active: id = %s, type = %d, internal = %d\n", id.ToString(), static_cast<int>(type), static_cast<int>(internal)); + auto& spk_mans = internal ? m_internal_spk_managers : m_external_spk_managers; + auto spk_man = m_spk_managers.at(id).get(); + spk_man->SetType(type, internal); + spk_mans[type] = spk_man; + + if (!memonly) { + WalletBatch batch(*database); + if (!batch.WriteActiveScriptPubKeyMan(static_cast<uint8_t>(type), id, internal)) { + throw std::runtime_error(std::string(__func__) + ": writing active ScriptPubKeyMan id failed"); + } + } + NotifyCanGetAddressesChanged(); +} + +bool CWallet::IsLegacy() const +{ + if (m_internal_spk_managers.count(OutputType::LEGACY) == 0) { + return false; + } + auto spk_man = dynamic_cast<LegacyScriptPubKeyMan*>(m_internal_spk_managers.at(OutputType::LEGACY)); + return spk_man != nullptr; +} + +DescriptorScriptPubKeyMan* CWallet::GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const +{ + for (auto& spk_man_pair : m_spk_managers) { + // Try to downcast to DescriptorScriptPubKeyMan then check if the descriptors match + DescriptorScriptPubKeyMan* spk_manager = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man_pair.second.get()); + if (spk_manager != nullptr && spk_manager->HasWalletDescriptor(desc)) { + return spk_manager; + } + } + + return nullptr; +} + +ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label) +{ + if (!IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + WalletLogPrintf("Cannot add WalletDescriptor to a non-descriptor wallet\n"); + return nullptr; + } + + LOCK(cs_wallet); + auto new_spk_man = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc)); + + // If we already have this descriptor, remove it from the maps but add the existing cache to desc + auto old_spk_man = GetDescriptorScriptPubKeyMan(desc); + if (old_spk_man) { + WalletLogPrintf("Update existing descriptor: %s\n", desc.descriptor->ToString()); + + { + LOCK(old_spk_man->cs_desc_man); + new_spk_man->SetCache(old_spk_man->GetWalletDescriptor().cache); + } + + // Remove from maps of active spkMans + auto old_spk_man_id = old_spk_man->GetID(); + for (bool internal : {false, true}) { + for (OutputType t : OUTPUT_TYPES) { + auto active_spk_man = GetScriptPubKeyMan(t, internal); + if (active_spk_man && active_spk_man->GetID() == old_spk_man_id) { + if (internal) { + m_internal_spk_managers.erase(t); + } else { + m_external_spk_managers.erase(t); + } + break; + } + } + } + m_spk_managers.erase(old_spk_man_id); + } + + // Add the private keys to the descriptor + for (const auto& entry : signing_provider.keys) { + const CKey& key = entry.second; + new_spk_man->AddDescriptorKey(key, key.GetPubKey()); + } + + // Top up key pool, the manager will generate new scriptPubKeys internally + new_spk_man->TopUp(); + + // Apply the label if necessary + // Note: we disable labels for ranged descriptors + if (!desc.descriptor->IsRange()) { + auto script_pub_keys = new_spk_man->GetScriptPubKeys(); + if (script_pub_keys.empty()) { + WalletLogPrintf("Could not generate scriptPubKeys (cache is empty)\n"); + return nullptr; + } + + CTxDestination dest; + if (ExtractDestination(script_pub_keys.at(0), dest)) { + SetAddressBook(dest, label, "receive"); + } + } + + // Save the descriptor to memory + auto ret = new_spk_man.get(); + m_spk_managers[new_spk_man->GetID()] = std::move(new_spk_man); + + // Save the descriptor to DB + ret->WriteDescriptor(); + + return ret; +} diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 176c483572..3be3435596 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -40,6 +40,8 @@ using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; +struct bilingual_str; + //! Explicitly unload and delete the wallet. //! Blocks the current thread after signaling the unload intent so that all //! wallet clients release the wallet. @@ -52,7 +54,7 @@ bool RemoveWallet(const std::shared_ptr<CWallet>& wallet); bool HasWallets(); std::vector<std::shared_ptr<CWallet>> GetWallets(); std::shared_ptr<CWallet> GetWallet(const std::string& name); -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::vector<std::string>& warnings); +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings); std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet); enum class WalletCreationStatus { @@ -61,7 +63,7 @@ enum class WalletCreationStatus { ENCRYPTION_FAILED }; -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result); +WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result); //! -paytxfee default constexpr CAmount DEFAULT_PAY_TX_FEE = 0; @@ -111,7 +113,8 @@ static constexpr uint64_t KNOWN_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE | WALLET_FLAG_BLANK_WALLET | WALLET_FLAG_KEY_ORIGIN_METADATA - | WALLET_FLAG_DISABLE_PRIVATE_KEYS; + | WALLET_FLAG_DISABLE_PRIVATE_KEYS + | WALLET_FLAG_DESCRIPTORS; static constexpr uint64_t MUTABLE_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE; @@ -121,6 +124,7 @@ static const std::map<std::string,WalletFlags> WALLET_FLAG_MAP{ {"blank", WALLET_FLAG_BLANK_WALLET}, {"key_origin_metadata", WALLET_FLAG_KEY_ORIGIN_METADATA}, {"disable_private_keys", WALLET_FLAG_DISABLE_PRIVATE_KEYS}, + {"descriptor_wallet", WALLET_FLAG_DESCRIPTORS}, }; extern const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS; @@ -497,8 +501,8 @@ public: bool IsEquivalentTo(const CWalletTx& tx) const; bool InMempool() const; - bool IsTrusted(interfaces::Chain::Lock& locked_chain) const; - bool IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const; + bool IsTrusted() const; + bool IsTrusted(std::set<uint256>& trusted_parents) const; int64_t GetTxTime() const; @@ -773,8 +777,8 @@ public: bool IsLocked() const override; bool Lock(); - /** Interface to assert chain access and if successful lock it */ - std::unique_ptr<interfaces::Chain::Lock> LockChain() { return m_chain ? m_chain->lock() : nullptr; } + /** Interface to assert chain access */ + bool HaveChain() const { return m_chain ? true : false; } std::map<uint256, CWalletTx> mapWallet GUARDED_BY(cs_wallet); @@ -803,12 +807,12 @@ public: /** * populate vCoins with vector of available COutputs. */ - void AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<COutput>& vCoins, bool fOnlySafe = true, const CCoinControl* coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe = true, const CCoinControl* coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Return list of available coins and locked coins grouped by non-change output address. */ - std::map<CTxDestination, std::vector<COutput>> ListCoins(interfaces::Chain::Lock& locked_chain) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + std::map<CTxDestination, std::vector<COutput>> ListCoins() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Find non-change parent output. @@ -873,7 +877,7 @@ public: bool ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, const SecureString& strNewWalletPassphrase); bool EncryptWallet(const SecureString& strWalletPassphrase); - void GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<CKeyID, int64_t> &mapKeyBirth) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void GetKeyBirthTimes(std::map<CKeyID, int64_t> &mapKeyBirth) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); unsigned int ComputeTimeSmart(const CWalletTx& wtx) const; /** @@ -928,7 +932,7 @@ public: * Insert additional inputs into the transaction by * calling CreateTransaction(); */ - bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); + bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); // Fetch the inputs and sign with SIGHASH_ALL. bool SignTransaction(CMutableTransaction& tx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); // Sign the tx given the input coins and sighash. @@ -959,8 +963,7 @@ public: * selected by SelectCoins(); Also create the change output, when needed * @note passing nChangePosInOut as -1 will result in setting a random position */ - bool CreateTransaction(interfaces::Chain::Lock& locked_chain, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, - std::string& strFailReason, const CCoinControl& coin_control, bool sign = true); + bool CreateTransaction(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, bool sign = true); /** * Submit the transaction to the node's mempool and then relay to peers. * Should be called after CreateTransaction unless you want to abort @@ -1010,7 +1013,7 @@ public: int64_t GetOldestKeyPoolTime() const; std::set<std::set<CTxDestination>> GetAddressGroupings() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::map<CTxDestination, CAmount> GetAddressBalances(interfaces::Chain::Lock& locked_chain) const; + std::map<CTxDestination, CAmount> GetAddressBalances() const; std::set<CTxDestination> GetLabelAddresses(const std::string& label) const; @@ -1123,10 +1126,10 @@ public: bool MarkReplaced(const uint256& originalHash, const uint256& newHash); //! Verify wallet naming and perform salvage on the wallet if required - static bool Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, std::string& error_string, std::vector<std::string>& warnings); + static bool Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, bilingual_str& error_string, std::vector<bilingual_str>& warnings); /* Initializes the wallet, returns a new CWallet instance or a null pointer in case of an error */ - static std::shared_ptr<CWallet> CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::vector<std::string>& warnings, uint64_t wallet_creation_flags = 0); + static std::shared_ptr<CWallet> CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags = 0); /** * Wallet post-init setup @@ -1163,6 +1166,9 @@ public: returns false if unknown, non-tolerable flags are present */ bool SetWalletFlags(uint64_t overwriteFlags, bool memOnly); + /** Determine if we are a legacy wallet */ + bool IsLegacy() const; + /** Returns a bracketed wallet name for displaying in logs, will return [default wallet] if the wallet has no name */ const std::string GetDisplayName() const override { std::string wallet_name = GetName().length() == 0 ? "default wallet" : GetName(); @@ -1176,7 +1182,7 @@ public: }; /** Upgrade the wallet */ - bool UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings); + bool UpgradeWallet(int version, bilingual_str& error, std::vector<bilingual_str>& warnings); //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; @@ -1232,6 +1238,25 @@ public: //! Connect the signals from ScriptPubKeyMans to the signals in CWallet void ConnectScriptPubKeyManNotifiers(); + + //! Instantiate a descriptor ScriptPubKeyMan from the WalletDescriptor and load it + void LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc); + + //! Sets the active ScriptPubKeyMan for the specified type and internal + //! @param[in] id The unique id for the ScriptPubKeyMan + //! @param[in] type The OutputType this ScriptPubKeyMan provides addresses for + //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses + //! @param[in] memonly Whether to record this update to the database. Set to true for wallet loading, normally false when actually updating the wallet. + void SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal, bool memonly = false); + + //! Create new DescriptorScriptPubKeyMans and add them to the wallet + void SetupDescriptorScriptPubKeyMans(); + + //! Return the DescriptorScriptPubKeyMan for a WalletDescriptor if it is already in the wallet + DescriptorScriptPubKeyMan* GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const; + + //! Add a descriptor to the wallet, return a ScriptPubKeyMan & associated output type + ScriptPubKeyMan* AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label); }; /** diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index f457db0a56..45033f9bac 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -21,6 +21,8 @@ namespace DBKeys { const std::string ACENTRY{"acentry"}; +const std::string ACTIVEEXTERNALSPK{"activeexternalspk"}; +const std::string ACTIVEINTERNALSPK{"activeinternalspk"}; const std::string BESTBLOCK_NOMERKLE{"bestblock_nomerkle"}; const std::string BESTBLOCK{"bestblock"}; const std::string CRYPTED_KEY{"ckey"}; @@ -41,6 +43,10 @@ const std::string PURPOSE{"purpose"}; const std::string SETTINGS{"settings"}; const std::string TX{"tx"}; const std::string VERSION{"version"}; +const std::string WALLETDESCRIPTOR{"walletdescriptor"}; +const std::string WALLETDESCRIPTORCACHE{"walletdescriptorcache"}; +const std::string WALLETDESCRIPTORCKEY{"walletdescriptorckey"}; +const std::string WALLETDESCRIPTORKEY{"walletdescriptorkey"}; const std::string WATCHMETA{"watchmeta"}; const std::string WATCHS{"watchs"}; } // namespace DBKeys @@ -179,6 +185,51 @@ bool WalletBatch::WriteMinVersion(int nVersion) return WriteIC(DBKeys::MINVERSION, nVersion); } +bool WalletBatch::WriteActiveScriptPubKeyMan(uint8_t type, const uint256& id, bool internal) +{ + std::string key = internal ? DBKeys::ACTIVEINTERNALSPK : DBKeys::ACTIVEEXTERNALSPK; + return WriteIC(make_pair(key, type), id); +} + +bool WalletBatch::WriteDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const CPrivKey& privkey) +{ + // hash pubkey/privkey to accelerate wallet load + std::vector<unsigned char> key; + key.reserve(pubkey.size() + privkey.size()); + key.insert(key.end(), pubkey.begin(), pubkey.end()); + key.insert(key.end(), privkey.begin(), privkey.end()); + + return WriteIC(std::make_pair(DBKeys::WALLETDESCRIPTORKEY, std::make_pair(desc_id, pubkey)), std::make_pair(privkey, Hash(key.begin(), key.end())), false); +} + +bool WalletBatch::WriteCryptedDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const std::vector<unsigned char>& secret) +{ + if (!WriteIC(std::make_pair(DBKeys::WALLETDESCRIPTORCKEY, std::make_pair(desc_id, pubkey)), secret, false)) { + return false; + } + EraseIC(std::make_pair(DBKeys::WALLETDESCRIPTORKEY, std::make_pair(desc_id, pubkey))); + return true; +} + +bool WalletBatch::WriteDescriptor(const uint256& desc_id, const WalletDescriptor& descriptor) +{ + return WriteIC(make_pair(DBKeys::WALLETDESCRIPTOR, desc_id), descriptor); +} + +bool WalletBatch::WriteDescriptorDerivedCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index, uint32_t der_index) +{ + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + xpub.Encode(ser_xpub.data()); + return WriteIC(std::make_pair(std::make_pair(DBKeys::WALLETDESCRIPTORCACHE, desc_id), std::make_pair(key_exp_index, der_index)), ser_xpub); +} + +bool WalletBatch::WriteDescriptorParentCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index) +{ + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + xpub.Encode(ser_xpub.data()); + return WriteIC(std::make_pair(std::make_pair(DBKeys::WALLETDESCRIPTORCACHE, desc_id), key_exp_index), ser_xpub); +} + class CWalletScanState { public: unsigned int nKeys{0}; @@ -189,6 +240,11 @@ public: bool fIsEncrypted{false}; bool fAnyUnordered{false}; std::vector<uint256> vWalletUpgrade; + std::map<OutputType, uint256> m_active_external_spks; + std::map<OutputType, uint256> m_active_internal_spks; + std::map<uint256, DescriptorCache> m_descriptor_caches; + std::map<std::pair<uint256, CKeyID>, CKey> m_descriptor_keys; + std::map<std::pair<uint256, CKeyID>, std::pair<CPubKey, std::vector<unsigned char>>> m_descriptor_crypt_keys; CWalletScanState() { } @@ -404,6 +460,108 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == DBKeys::OLD_KEY) { strErr = "Found unsupported 'wkey' record, try loading with version 0.18"; return false; + } else if (strType == DBKeys::ACTIVEEXTERNALSPK || strType == DBKeys::ACTIVEINTERNALSPK) { + uint8_t type; + ssKey >> type; + uint256 id; + ssValue >> id; + + bool internal = strType == DBKeys::ACTIVEINTERNALSPK; + auto& spk_mans = internal ? wss.m_active_internal_spks : wss.m_active_external_spks; + if (spk_mans.count(static_cast<OutputType>(type)) > 0) { + strErr = "Multiple ScriptPubKeyMans specified for a single type"; + return false; + } + spk_mans[static_cast<OutputType>(type)] = id; + } else if (strType == DBKeys::WALLETDESCRIPTOR) { + uint256 id; + ssKey >> id; + WalletDescriptor desc; + ssValue >> desc; + if (wss.m_descriptor_caches.count(id) == 0) { + wss.m_descriptor_caches[id] = DescriptorCache(); + } + pwallet->LoadDescriptorScriptPubKeyMan(id, desc); + } else if (strType == DBKeys::WALLETDESCRIPTORCACHE) { + bool parent = true; + uint256 desc_id; + uint32_t key_exp_index; + uint32_t der_index; + ssKey >> desc_id; + ssKey >> key_exp_index; + + // if the der_index exists, it's a derived xpub + try + { + ssKey >> der_index; + parent = false; + } + catch (...) {} + + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + ssValue >> ser_xpub; + CExtPubKey xpub; + xpub.Decode(ser_xpub.data()); + if (wss.m_descriptor_caches.count(desc_id)) { + wss.m_descriptor_caches[desc_id] = DescriptorCache(); + } + if (parent) { + wss.m_descriptor_caches[desc_id].CacheParentExtPubKey(key_exp_index, xpub); + } else { + wss.m_descriptor_caches[desc_id].CacheDerivedExtPubKey(key_exp_index, der_index, xpub); + } + } else if (strType == DBKeys::WALLETDESCRIPTORKEY) { + uint256 desc_id; + CPubKey pubkey; + ssKey >> desc_id; + ssKey >> pubkey; + if (!pubkey.IsValid()) + { + strErr = "Error reading wallet database: CPubKey corrupt"; + return false; + } + CKey key; + CPrivKey pkey; + uint256 hash; + + wss.nKeys++; + ssValue >> pkey; + ssValue >> hash; + + // hash pubkey/privkey to accelerate wallet load + std::vector<unsigned char> to_hash; + to_hash.reserve(pubkey.size() + pkey.size()); + to_hash.insert(to_hash.end(), pubkey.begin(), pubkey.end()); + to_hash.insert(to_hash.end(), pkey.begin(), pkey.end()); + + if (Hash(to_hash.begin(), to_hash.end()) != hash) + { + strErr = "Error reading wallet database: CPubKey/CPrivKey corrupt"; + return false; + } + + if (!key.Load(pkey, pubkey, true)) + { + strErr = "Error reading wallet database: CPrivKey corrupt"; + return false; + } + wss.m_descriptor_keys.insert(std::make_pair(std::make_pair(desc_id, pubkey.GetID()), key)); + } else if (strType == DBKeys::WALLETDESCRIPTORCKEY) { + uint256 desc_id; + CPubKey pubkey; + ssKey >> desc_id; + ssKey >> pubkey; + if (!pubkey.IsValid()) + { + strErr = "Error reading wallet database: CPubKey corrupt"; + return false; + } + std::vector<unsigned char> privkey; + ssValue >> privkey; + wss.nCKeys++; + + wss.m_descriptor_crypt_keys.insert(std::make_pair(std::make_pair(desc_id, pubkey.GetID()), std::make_pair(pubkey, privkey))); + wss.fIsEncrypted = true; } else if (strType != DBKeys::BESTBLOCK && strType != DBKeys::BESTBLOCK_NOMERKLE && strType != DBKeys::MINVERSION && strType != DBKeys::ACENTRY && strType != DBKeys::VERSION && strType != DBKeys::SETTINGS) { @@ -497,6 +655,31 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) result = DBErrors::CORRUPT; } + // Set the active ScriptPubKeyMans + for (auto spk_man_pair : wss.m_active_external_spks) { + pwallet->SetActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ false, /* memonly */ true); + } + for (auto spk_man_pair : wss.m_active_internal_spks) { + pwallet->SetActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ true, /* memonly */ true); + } + + // Set the descriptor caches + for (auto desc_cache_pair : wss.m_descriptor_caches) { + auto spk_man = pwallet->GetScriptPubKeyMan(desc_cache_pair.first); + assert(spk_man); + ((DescriptorScriptPubKeyMan*)spk_man)->SetCache(desc_cache_pair.second); + } + + // Set the descriptor keys + for (auto desc_key_pair : wss.m_descriptor_keys) { + auto spk_man = pwallet->GetScriptPubKeyMan(desc_key_pair.first.first); + ((DescriptorScriptPubKeyMan*)spk_man)->AddKey(desc_key_pair.first.second, desc_key_pair.second); + } + for (auto desc_key_pair : wss.m_descriptor_crypt_keys) { + auto spk_man = pwallet->GetScriptPubKeyMan(desc_key_pair.first.first); + ((DescriptorScriptPubKeyMan*)spk_man)->AddCryptedKey(desc_key_pair.first.second, desc_key_pair.second.first, desc_key_pair.second.second); + } + if (fNoncriticalErrors && result == DBErrors::LOAD_OK) result = DBErrors::NONCRITICAL_ERROR; @@ -516,7 +699,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys, wss.m_unknown_records); // nTimeFirstKey is only reliable if all keys have metadata - if ((wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { + if (pwallet->IsLegacy() && (wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { auto spk_man = pwallet->GetOrCreateLegacyScriptPubKeyMan(); if (spk_man) { LOCK(spk_man->cs_KeyStore); @@ -730,12 +913,12 @@ bool WalletBatch::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, C return true; } -bool WalletBatch::VerifyEnvironment(const fs::path& wallet_path, std::string& errorStr) +bool WalletBatch::VerifyEnvironment(const fs::path& wallet_path, bilingual_str& errorStr) { return BerkeleyBatch::VerifyEnvironment(wallet_path, errorStr); } -bool WalletBatch::VerifyDatabaseFile(const fs::path& wallet_path, std::vector<std::string>& warnings, std::string& errorStr) +bool WalletBatch::VerifyDatabaseFile(const fs::path& wallet_path, std::vector<bilingual_str>& warnings, bilingual_str& errorStr) { return BerkeleyBatch::VerifyDatabaseFile(wallet_path, warnings, errorStr, WalletBatch::Recover); } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index a436f60ab3..ae72a5b265 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -9,6 +9,7 @@ #include <amount.h> #include <script/sign.h> #include <wallet/db.h> +#include <wallet/walletutil.h> #include <key.h> #include <stdint.h> @@ -54,6 +55,8 @@ enum class DBErrors namespace DBKeys { extern const std::string ACENTRY; +extern const std::string ACTIVEEXTERNALSPK; +extern const std::string ACTIVEINTERNALSPK; extern const std::string BESTBLOCK; extern const std::string BESTBLOCK_NOMERKLE; extern const std::string CRYPTED_KEY; @@ -74,6 +77,9 @@ extern const std::string PURPOSE; extern const std::string SETTINGS; extern const std::string TX; extern const std::string VERSION; +extern const std::string WALLETDESCRIPTOR; +extern const std::string WALLETDESCRIPTORCKEY; +extern const std::string WALLETDESCRIPTORKEY; extern const std::string WATCHMETA; extern const std::string WATCHS; } // namespace DBKeys @@ -240,11 +246,19 @@ public: bool WriteMinVersion(int nVersion); + bool WriteDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const CPrivKey& privkey); + bool WriteCryptedDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const std::vector<unsigned char>& secret); + bool WriteDescriptor(const uint256& desc_id, const WalletDescriptor& descriptor); + bool WriteDescriptorDerivedCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index, uint32_t der_index); + bool WriteDescriptorParentCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index); + /// Write destination data key,value tuple to database bool WriteDestData(const std::string &address, const std::string &key, const std::string &value); /// Erase destination data tuple from wallet database bool EraseDestData(const std::string &address, const std::string &key); + bool WriteActiveScriptPubKeyMan(uint8_t type, const uint256& id, bool internal); + DBErrors LoadWallet(CWallet* pwallet); DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx); DBErrors ZapWalletTx(std::vector<CWalletTx>& vWtx); @@ -258,9 +272,9 @@ public: /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ static bool IsKeyType(const std::string& strType); /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path& wallet_path, std::string& errorStr); + static bool VerifyEnvironment(const fs::path& wallet_path, bilingual_str& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& wallet_path, std::vector<std::string>& warnings, std::string& errorStr); + static bool VerifyDatabaseFile(const fs::path& wallet_path, std::vector<bilingual_str>& warnings, bilingual_str& errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 65643669c2..522efaa884 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -4,6 +4,7 @@ #include <fs.h> #include <util/system.h> +#include <util/translation.h> #include <wallet/wallet.h> #include <wallet/walletutil.h> @@ -117,9 +118,9 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) tfm::format(std::cerr, "Error: no wallet file at %s\n", name); return false; } - std::string error; + bilingual_str error; if (!WalletBatch::VerifyEnvironment(path, error)) { - tfm::format(std::cerr, "Error loading %s. Is wallet being used by other process?\n", name); + tfm::format(std::cerr, "%s\nError loading %s. Is wallet being used by other process?\n", error.original, name); return false; } std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 04c2407a89..8bac0608a9 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -100,5 +100,10 @@ WalletLocation::WalletLocation(const std::string& name) bool WalletLocation::Exists() const { - return fs::symlink_status(m_path).type() != fs::file_not_found; + fs::path path = m_path; + // For the default wallet, check specifically for the wallet.dat file + if (m_name.empty()) { + path = fs::absolute("wallet.dat", m_path); + } + return fs::symlink_status(path).type() != fs::file_not_found; } diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index c91c9aca96..599b1a9f5a 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -6,6 +6,7 @@ #define BITCOIN_WALLET_WALLETUTIL_H #include <fs.h> +#include <script/descriptor.h> #include <vector> @@ -55,6 +56,9 @@ enum WalletFlags : uint64_t { //! bitcoin from opening the wallet, thinking it was newly created, and //! then improperly reinitializing it. WALLET_FLAG_BLANK_WALLET = (1ULL << 33), + + //! Indicate that this wallet supports DescriptorScriptPubKeyMan + WALLET_FLAG_DESCRIPTORS = (1ULL << 34), }; //! Get the path of the wallet directory. @@ -83,4 +87,41 @@ public: bool Exists() const; }; +/** Descriptor with some wallet metadata */ +class WalletDescriptor +{ +public: + std::shared_ptr<Descriptor> descriptor; + uint64_t creation_time = 0; + int32_t range_start = 0; // First item in range; start of range, inclusive, i.e. [range_start, range_end). This never changes. + int32_t range_end = 0; // Item after the last; end of range, exclusive, i.e. [range_start, range_end). This will increment with each TopUp() + int32_t next_index = 0; // Position of the next item to generate + DescriptorCache cache; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + if (ser_action.ForRead()) { + std::string desc; + std::string error; + READWRITE(desc); + FlatSigningProvider keys; + descriptor = Parse(desc, keys, error, true); + if (!descriptor) { + throw std::ios_base::failure("Invalid descriptor: " + error); + } + } else { + READWRITE(descriptor->ToString()); + } + READWRITE(creation_time); + READWRITE(next_index); + READWRITE(range_start); + READWRITE(range_end); + } + + WalletDescriptor() {} + WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) {} +}; + #endif // BITCOIN_WALLET_WALLETUTIL_H diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index 75e0b93c83..9cff79a42c 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -18,15 +18,16 @@ needs an older patch version. import os import shutil -from test_framework.test_framework import BitcoinTestFramework, SkipTest +from test_framework.test_framework import BitcoinTestFramework from test_framework.descriptors import descsum_create from test_framework.util import ( assert_equal, sync_blocks, - sync_mempools + sync_mempools, ) + class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -42,35 +43,15 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_previous_releases() def setup_nodes(self): - if os.getenv("TEST_PREVIOUS_RELEASES") == "false": - raise SkipTest("backwards compatibility tests") - - releases_path = os.getenv("PREVIOUS_RELEASES_DIR") or os.getcwd() + "/releases" - if not os.path.isdir(releases_path): - if os.getenv("TEST_PREVIOUS_RELEASES") == "true": - raise AssertionError("TEST_PREVIOUS_RELEASES=1 but releases missing: " + releases_path) - raise SkipTest("This test requires binaries for previous releases") - self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ None, None, - 190000, + 190001, 180100, - 170100 - ], binary=[ - self.options.bitcoind, - self.options.bitcoind, - releases_path + "/v0.19.0.1/bin/bitcoind", - releases_path + "/v0.18.1/bin/bitcoind", - releases_path + "/v0.17.1/bin/bitcoind" - ], binary_cli=[ - self.options.bitcoincli, - self.options.bitcoincli, - releases_path + "/v0.19.0.1/bin/bitcoin-cli", - releases_path + "/v0.18.1/bin/bitcoin-cli", - releases_path + "/v0.17.1/bin/bitcoin-cli" + 170100, ]) self.start_nodes() @@ -94,7 +75,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # w1: regular wallet, created on master: update this test when default # wallets can no longer be opened by older versions. - node_master.createwallet(wallet_name="w1") + node_master.rpc.createwallet(wallet_name="w1") wallet = node_master.get_wallet_rpc("w1") info = wallet.getwalletinfo() assert info['private_keys_enabled'] @@ -120,17 +101,17 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.nodes[1].abandontransaction(tx3_id) # w1_v19: regular wallet, created with v0.19 - node_v19.createwallet(wallet_name="w1_v19") + node_v19.rpc.createwallet(wallet_name="w1_v19") wallet = node_v19.get_wallet_rpc("w1_v19") info = wallet.getwalletinfo() assert info['private_keys_enabled'] assert info['keypoolsize'] > 0 # Use addmultisigaddress (see #18075) - address_18075 = wallet.addmultisigaddress(1, ["0296b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52", "037211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073"], "", "legacy")["address"] + address_18075 = wallet.rpc.addmultisigaddress(1, ["0296b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52", "037211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073"], "", "legacy")["address"] assert wallet.getaddressinfo(address_18075)["solvable"] # w1_v18: regular wallet, created with v0.18 - node_v18.createwallet(wallet_name="w1_v18") + node_v18.rpc.createwallet(wallet_name="w1_v18") wallet = node_v18.get_wallet_rpc("w1_v18") info = wallet.getwalletinfo() assert info['private_keys_enabled'] @@ -139,21 +120,21 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # w2: wallet with private keys disabled, created on master: update this # test when default wallets private keys disabled can no longer be # opened by older versions. - node_master.createwallet(wallet_name="w2", disable_private_keys=True) + node_master.rpc.createwallet(wallet_name="w2", disable_private_keys=True) wallet = node_master.get_wallet_rpc("w2") info = wallet.getwalletinfo() assert info['private_keys_enabled'] == False assert info['keypoolsize'] == 0 # w2_v19: wallet with private keys disabled, created with v0.19 - node_v19.createwallet(wallet_name="w2_v19", disable_private_keys=True) + node_v19.rpc.createwallet(wallet_name="w2_v19", disable_private_keys=True) wallet = node_v19.get_wallet_rpc("w2_v19") info = wallet.getwalletinfo() assert info['private_keys_enabled'] == False assert info['keypoolsize'] == 0 # w2_v18: wallet with private keys disabled, created with v0.18 - node_v18.createwallet(wallet_name="w2_v18", disable_private_keys=True) + node_v18.rpc.createwallet(wallet_name="w2_v18", disable_private_keys=True) wallet = node_v18.get_wallet_rpc("w2_v18") info = wallet.getwalletinfo() assert info['private_keys_enabled'] == False @@ -161,21 +142,21 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # w3: blank wallet, created on master: update this # test when default blank wallets can no longer be opened by older versions. - node_master.createwallet(wallet_name="w3", blank=True) + node_master.rpc.createwallet(wallet_name="w3", blank=True) wallet = node_master.get_wallet_rpc("w3") info = wallet.getwalletinfo() assert info['private_keys_enabled'] assert info['keypoolsize'] == 0 # w3_v19: blank wallet, created with v0.19 - node_v19.createwallet(wallet_name="w3_v19", blank=True) + node_v19.rpc.createwallet(wallet_name="w3_v19", blank=True) wallet = node_v19.get_wallet_rpc("w3_v19") info = wallet.getwalletinfo() assert info['private_keys_enabled'] assert info['keypoolsize'] == 0 # w3_v18: blank wallet, created with v0.18 - node_v18.createwallet(wallet_name="w3_v18", blank=True) + node_v18.rpc.createwallet(wallet_name="w3_v18", blank=True) wallet = node_v18.get_wallet_rpc("w3_v18") info = wallet.getwalletinfo() assert info['private_keys_enabled'] @@ -318,12 +299,12 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.log.info("Test wallet upgrade path...") # u1: regular wallet, created with v0.17 - node_v17.createwallet(wallet_name="u1_v17") + node_v17.rpc.createwallet(wallet_name="u1_v17") wallet = node_v17.get_wallet_rpc("u1_v17") address = wallet.getnewaddress("bech32") - info = wallet.getaddressinfo(address) - hdkeypath = info["hdkeypath"] - pubkey = info["pubkey"] + v17_info = wallet.getaddressinfo(address) + hdkeypath = v17_info["hdkeypath"] + pubkey = v17_info["pubkey"] # Copy the 0.17 wallet to the last Bitcoin Core version and open it: node_v17.unloadwallet("u1_v17") @@ -337,6 +318,18 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")" assert_equal(info["desc"], descsum_create(descriptor)) + # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it + node_master.unloadwallet("u1_v17") + shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "u1_v17"), + os.path.join(node_v17_wallets_dir, "u1_v17") + ) + node_v17.loadwallet("u1_v17") + wallet = node_v17.get_wallet_rpc("u1_v17") + info = wallet.getaddressinfo(address) + assert_equal(info, v17_info) + # Copy the 0.19 wallet to the last Bitcoin Core version and open it: shutil.copytree( os.path.join(node_v19_wallets_dir, "w1_v19"), @@ -346,5 +339,16 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): wallet = node_master.get_wallet_rpc("w1_v19") assert wallet.getaddressinfo(address_18075)["solvable"] + # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it + node_master.unloadwallet("w1_v19") + shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "w1_v19"), + os.path.join(node_v19_wallets_dir, "w1_v19") + ) + node_v19.loadwallet("w1_v19") + wallet = node_v19.get_wallet_rpc("w1_v19") + assert wallet.getaddressinfo(address_18075)["solvable"] + if __name__ == '__main__': BackwardsCompatibilityTest().main() diff --git a/test/functional/framework_test_script.py b/test/functional/framework_test_script.py deleted file mode 100755 index 9d916c0022..0000000000 --- a/test/functional/framework_test_script.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Tests for test_framework.script.""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.script import bn2vch -from test_framework.util import assert_equal - -def test_bn2vch(): - assert_equal(bn2vch(0), bytes([])) - assert_equal(bn2vch(1), bytes([0x01])) - assert_equal(bn2vch(-1), bytes([0x81])) - assert_equal(bn2vch(0x7F), bytes([0x7F])) - assert_equal(bn2vch(-0x7F), bytes([0xFF])) - assert_equal(bn2vch(0x80), bytes([0x80, 0x00])) - assert_equal(bn2vch(-0x80), bytes([0x80, 0x80])) - assert_equal(bn2vch(0xFF), bytes([0xFF, 0x00])) - assert_equal(bn2vch(-0xFF), bytes([0xFF, 0x80])) - assert_equal(bn2vch(0x100), bytes([0x00, 0x01])) - assert_equal(bn2vch(-0x100), bytes([0x00, 0x81])) - assert_equal(bn2vch(0x7FFF), bytes([0xFF, 0x7F])) - assert_equal(bn2vch(-0x8000), bytes([0x00, 0x80, 0x80])) - assert_equal(bn2vch(-0x7FFFFF), bytes([0xFF, 0xFF, 0xFF])) - assert_equal(bn2vch(0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x00])) - assert_equal(bn2vch(-0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x80])) - assert_equal(bn2vch(0xFFFFFFFF), bytes([0xFF, 0xFF, 0xFF, 0xFF, 0x00])) - - assert_equal(bn2vch(123456789), bytes([0x15, 0xCD, 0x5B, 0x07])) - assert_equal(bn2vch(-54321), bytes([0x31, 0xD4, 0x80])) - -class FrameworkTestScript(BitcoinTestFramework): - def setup_network(self): - pass - - def set_test_params(self): - self.num_nodes = 0 - - def run_test(self): - test_bn2vch() - -if __name__ == '__main__': - FrameworkTestScript().main() diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index e1671624a8..99003d2d1f 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -40,10 +40,13 @@ import os import time from test_framework.test_framework import BitcoinTestFramework +from test_framework.mininode import P2PTxInvStore from test_framework.util import ( assert_equal, assert_greater_than_or_equal, assert_raises_rpc_error, + connect_nodes, + disconnect_nodes, wait_until, ) @@ -80,6 +83,11 @@ class MempoolPersistTest(BitcoinTestFramework): assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower) assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) + # disconnect nodes & make a txn that remains in the unbroadcast set. + disconnect_nodes(self.nodes[0], 2) + self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("12")) + connect_nodes(self.nodes[0], 2) + self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.") self.stop_nodes() # Give this node a head-start, so we can be "extra-sure" that it didn't load anything later @@ -89,7 +97,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.start_node(2) wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"], timeout=1) wait_until(lambda: self.nodes[2].getmempoolinfo()["loaded"], timeout=1) - assert_equal(len(self.nodes[0].getrawmempool()), 5) + assert_equal(len(self.nodes[0].getrawmempool()), 6) assert_equal(len(self.nodes[2].getrawmempool()), 5) # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now: assert_equal(len(self.nodes[1].getrawmempool()), 0) @@ -105,9 +113,10 @@ class MempoolPersistTest(BitcoinTestFramework): self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(node2_balance, self.nodes[2].getbalance()) + # start node0 with wallet disabled so wallet transactions don't get resubmitted self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() - self.start_node(0, extra_args=["-persistmempool=0"]) + self.start_node(0, extra_args=["-persistmempool=0", "-disablewallet"]) wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -115,7 +124,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.stop_nodes() self.start_node(0) wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) - assert_equal(len(self.nodes[0].getrawmempool()), 5) + assert_equal(len(self.nodes[0].getrawmempool()), 6) mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') @@ -124,12 +133,12 @@ class MempoolPersistTest(BitcoinTestFramework): self.nodes[0].savemempool() assert os.path.isfile(mempooldat0) - self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 5 transactions") + self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 6 transactions") os.rename(mempooldat0, mempooldat1) self.stop_nodes() self.start_node(1, extra_args=[]) wait_until(lambda: self.nodes[1].getmempoolinfo()["loaded"]) - assert_equal(len(self.nodes[1].getrawmempool()), 5) + assert_equal(len(self.nodes[1].getrawmempool()), 6) self.log.debug("Prevent bitcoind from writing mempool.dat to disk. Verify that `savemempool` fails") # to test the exception we are creating a tmp folder called mempool.dat.new @@ -139,6 +148,27 @@ class MempoolPersistTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "Unable to dump mempool to disk", self.nodes[1].savemempool) os.rmdir(mempooldotnew1) + self.test_persist_unbroadcast() + + def test_persist_unbroadcast(self): + node0 = self.nodes[0] + self.start_node(0) + + # clear out mempool + node0.generate(1) + + # disconnect nodes to make a txn that remains in the unbroadcast set. + disconnect_nodes(node0, 1) + node0.sendtoaddress(self.nodes[1].getnewaddress(), Decimal("12")) + + # shutdown, then startup with wallet disabled + self.stop_nodes() + self.start_node(0, extra_args=["-disablewallet"]) + + # check that txn gets broadcast due to unbroadcast logic + conn = node0.add_p2p_connection(P2PTxInvStore()) + node0.mockscheduler(16*60) # 15 min + 1 for buffer + wait_until(lambda: len(conn.get_invs()) == 1) if __name__ == '__main__': MempoolPersistTest().main() diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py new file mode 100755 index 0000000000..a561f28b91 --- /dev/null +++ b/test/functional/mempool_unbroadcast.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that the mempool ensures transaction delivery by periodically sending +to peers until a GETDATA is received.""" + +import time + +from test_framework.mininode import P2PTxInvStore +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + connect_nodes, + create_confirmed_utxos, + disconnect_nodes, +) + + +class MempoolUnbroadcastTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.test_broadcast() + self.test_txn_removal() + + def test_broadcast(self): + self.log.info("Test that mempool reattempts delivery of locally submitted transaction") + node = self.nodes[0] + + min_relay_fee = node.getnetworkinfo()["relayfee"] + utxos = create_confirmed_utxos(min_relay_fee, node, 10) + + disconnect_nodes(node, 1) + + self.log.info("Generate transactions that only node 0 knows about") + + # generate a wallet txn + addr = node.getnewaddress() + wallet_tx_hsh = node.sendtoaddress(addr, 0.0001) + + # generate a txn using sendrawtransaction + us0 = utxos.pop() + inputs = [{"txid": us0["txid"], "vout": us0["vout"]}] + outputs = {addr: 0.0001} + tx = node.createrawtransaction(inputs, outputs) + node.settxfee(min_relay_fee) + txF = node.fundrawtransaction(tx) + txFS = node.signrawtransactionwithwallet(txF["hex"]) + rpc_tx_hsh = node.sendrawtransaction(txFS["hex"]) + + # check that second node doesn't have these two txns + mempool = self.nodes[1].getrawmempool() + assert rpc_tx_hsh not in mempool + assert wallet_tx_hsh not in mempool + + # ensure that unbroadcast txs are persisted to mempool.dat + self.restart_node(0) + + self.log.info("Reconnect nodes & check if they are sent to node 1") + connect_nodes(node, 1) + + # fast forward into the future & ensure that the second node has the txns + node.mockscheduler(15 * 60) # 15 min in seconds + self.sync_mempools(timeout=30) + mempool = self.nodes[1].getrawmempool() + assert rpc_tx_hsh in mempool + assert wallet_tx_hsh in mempool + + self.log.info("Add another connection & ensure transactions aren't broadcast again") + + conn = node.add_p2p_connection(P2PTxInvStore()) + node.mockscheduler(15 * 60) + time.sleep(5) + assert_equal(len(conn.get_invs()), 0) + + def test_txn_removal(self): + self.log.info("Test that transactions removed from mempool are removed from unbroadcast set") + node = self.nodes[0] + disconnect_nodes(node, 1) + node.disconnect_p2ps + + # since the node doesn't have any connections, it will not receive + # any GETDATAs & thus the transaction will remain in the unbroadcast set. + addr = node.getnewaddress() + txhsh = node.sendtoaddress(addr, 0.0001) + + # check transaction was removed from unbroadcast set due to presence in + # a block + removal_reason = "Removed {} from set of unbroadcast txns before confirmation that txn was sent out".format(txhsh) + with node.assert_debug_log([removal_reason]): + node.generate(1) + +if __name__ == "__main__": + MempoolUnbroadcastTest().main() diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py new file mode 100755 index 0000000000..8a703ef009 --- /dev/null +++ b/test/functional/mempool_updatefromblock.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test mempool descendants/ancestors information update. + +Test mempool update of transaction descendants/ancestors information (count, size) +when transactions have been re-added from a disconnected block to the mempool. +""" +import time + +from decimal import Decimal +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +class MempoolUpdateFromBlockTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [['-limitdescendantsize=1000', '-limitancestorsize=1000']] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', end_address='', fee=Decimal(0.00100000)): + """Create an acyclic tournament (a type of directed graph) of transactions and use it for testing. + + Keyword arguments: + size -- the order N of the tournament which is equal to the number of the created transactions + n_tx_to_mine -- the number of transaction that should be mined into a block + + If all of the N created transactions tx[0]..tx[N-1] reside in the mempool, + the following holds: + the tx[K] transaction: + - has N-K descendants (including this one), and + - has K+1 ancestors (including this one) + + More details: https://en.wikipedia.org/wiki/Tournament_(graph_theory) + """ + + if not start_input_txid: + start_input_txid = self.nodes[0].getblock(self.nodes[0].getblockhash(1))['tx'][0] + + if not end_address: + end_address = self.nodes[0].getnewaddress() + + first_block_hash = '' + tx_id = [] + tx_size = [] + self.log.info('Creating {} transactions...'.format(size)) + for i in range(0, size): + self.log.debug('Preparing transaction #{}...'.format(i)) + # Prepare inputs. + if i == 0: + inputs = [{'txid': start_input_txid, 'vout': 0}] + inputs_value = self.nodes[0].gettxout(start_input_txid, 0)['value'] + else: + inputs = [] + inputs_value = 0 + for j, tx in enumerate(tx_id[0:i]): + # Transaction tx[K] is a child of each of previous transactions tx[0]..tx[K-1] at their output K-1. + vout = i - j - 1 + inputs.append({'txid': tx_id[j], 'vout': vout}) + inputs_value += self.nodes[0].gettxout(tx, vout)['value'] + + self.log.debug('inputs={}'.format(inputs)) + self.log.debug('inputs_value={}'.format(inputs_value)) + + # Prepare outputs. + tx_count = i + 1 + if tx_count < size: + # Transaction tx[K] is an ancestor of each of subsequent transactions tx[K+1]..tx[N-1]. + n_outputs = size - tx_count + output_value = ((inputs_value - fee) / Decimal(n_outputs)).quantize(Decimal('0.00000001')) + outputs = {} + for n in range(0, n_outputs): + outputs[self.nodes[0].getnewaddress()] = output_value + else: + output_value = (inputs_value - fee).quantize(Decimal('0.00000001')) + outputs = {end_address: output_value} + + self.log.debug('output_value={}'.format(output_value)) + self.log.debug('outputs={}'.format(outputs)) + + # Create a new transaction. + unsigned_raw_tx = self.nodes[0].createrawtransaction(inputs, outputs) + signed_raw_tx = self.nodes[0].signrawtransactionwithwallet(unsigned_raw_tx) + tx_id.append(self.nodes[0].sendrawtransaction(signed_raw_tx['hex'])) + tx_size.append(self.nodes[0].getrawmempool(True)[tx_id[-1]]['vsize']) + + if tx_count in n_tx_to_mine: + # The created transactions are mined into blocks by batches. + self.log.info('The batch of {} transactions has been accepted into the mempool.'.format(len(self.nodes[0].getrawmempool()))) + block_hash = self.nodes[0].generate(1)[0] + if not first_block_hash: + first_block_hash = block_hash + assert_equal(len(self.nodes[0].getrawmempool()), 0) + self.log.info('All of the transactions from the current batch have been mined into a block.') + elif tx_count == size: + # At the end all of the mined blocks are invalidated, and all of the created + # transactions should be re-added from disconnected blocks to the mempool. + self.log.info('The last batch of {} transactions has been accepted into the mempool.'.format(len(self.nodes[0].getrawmempool()))) + start = time.time() + self.nodes[0].invalidateblock(first_block_hash) + end = time.time() + assert_equal(len(self.nodes[0].getrawmempool()), size) + self.log.info('All of the recently mined transactions have been re-added into the mempool in {} seconds.'.format(end - start)) + + self.log.info('Checking descendants/ancestors properties of all of the in-mempool transactions...') + for k, tx in enumerate(tx_id): + self.log.debug('Check transaction #{}.'.format(k)) + assert_equal(self.nodes[0].getrawmempool(True)[tx]['descendantcount'], size - k) + assert_equal(self.nodes[0].getrawmempool(True)[tx]['descendantsize'], sum(tx_size[k:size])) + assert_equal(self.nodes[0].getrawmempool(True)[tx]['ancestorcount'], k + 1) + assert_equal(self.nodes[0].getrawmempool(True)[tx]['ancestorsize'], sum(tx_size[0:(k + 1)])) + + def run_test(self): + # Use batch size limited by DEFAULT_ANCESTOR_LIMIT = 25 to not fire "too many unconfirmed parents" error. + self.transaction_graph_test(size=100, n_tx_to_mine=[25, 50, 75]) + + +if __name__ == '__main__': + MempoolUpdateFromBlockTest().main() diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index a8b768c144..15955a938c 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -64,19 +64,40 @@ class FilterTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def run_test(self): - filter_node = self.nodes[0].add_p2p_connection(FilterNode()) - + def test_size_limits(self, filter_node): self.log.info('Check that too large filter is rejected') with self.nodes[0].assert_debug_log(['Misbehaving']): - filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS+1)) - with self.nodes[0].assert_debug_log(['Misbehaving']): filter_node.send_and_ping(msg_filterload(data=b'\xbb'*(MAX_BLOOM_FILTER_SIZE+1))) + self.log.info('Check that max size filter is accepted') + with self.nodes[0].assert_debug_log([], unexpected_msgs=['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xbb'*(MAX_BLOOM_FILTER_SIZE))) + filter_node.send_and_ping(msg_filterclear()) + + self.log.info('Check that filter with too many hash functions is rejected') + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS+1)) + + self.log.info('Check that filter with max hash functions is accepted') + with self.nodes[0].assert_debug_log([], unexpected_msgs=['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS)) + # Don't send filterclear until next two filteradd checks are done + + self.log.info('Check that max size data element to add to the filter is accepted') + with self.nodes[0].assert_debug_log([], unexpected_msgs=['Misbehaving']): + filter_node.send_and_ping(msg_filteradd(data=b'\xcc'*(MAX_SCRIPT_ELEMENT_SIZE))) + self.log.info('Check that too large data element to add to the filter is rejected') with self.nodes[0].assert_debug_log(['Misbehaving']): filter_node.send_and_ping(msg_filteradd(data=b'\xcc'*(MAX_SCRIPT_ELEMENT_SIZE+1))) + filter_node.send_and_ping(msg_filterclear()) + + def run_test(self): + filter_node = self.nodes[0].add_p2p_connection(FilterNode()) + + self.test_size_limits(filter_node) + self.log.info('Add filtered P2P connection to the node') filter_node.send_and_ping(filter_node.watch_filter_init) filter_address = self.nodes[0].decodescript(filter_node.watch_script_pubkey)['addresses'][0] diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 7f7430d04e..157af68203 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -141,12 +141,11 @@ class P2PLeakTest(BitcoinTestFramework): assert no_verack_idlenode.unexpected_msg == False self.log.info('Check that the version message does not leak the local address of the node') - time_begin = int(time.time()) p2p_version_store = self.nodes[0].add_p2p_connection(P2PVersionStore()) - time_end = time.time() ver = p2p_version_store.version_received - assert_greater_than_or_equal(ver.nTime, time_begin) - assert_greater_than_or_equal(time_end, ver.nTime) + # Check that received time is within one hour of now + assert_greater_than_or_equal(ver.nTime, time.time() - 3600) + assert_greater_than_or_equal(time.time() + 3600, ver.nTime) assert_equal(ver.addrFrom.port, 0) assert_equal(ver.addrFrom.ip, '0.0.0.0') assert_equal(ver.nStartingHeight, 201) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index dbdce6552a..6fb0fec32b 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1896,12 +1896,12 @@ class SegWitTest(BitcoinTestFramework): def test_upgrade_after_activation(self): """Test the behavior of starting up a segwit-aware node after the softfork has activated.""" - # Restart with the new binary self.stop_node(2) self.start_node(2, extra_args=["-segwitheight={}".format(SEGWIT_HEIGHT)]) connect_nodes(self.nodes[0], 2) - self.sync_blocks() + # We reconnect more than 100 blocks, give it plenty of time + self.sync_blocks(timeout=240) # Make sure that this peer thinks segwit has activated. assert softfork_active(self.nodes[2], 'segwit') diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index a983716177..56e9ecfcc2 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -4,13 +4,14 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multisig RPCs""" +from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create, drop_origins from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_raises_rpc_error, assert_equal, ) -from test_framework.key import ECPubKey +from test_framework.key import ECPubKey, ECKey, bytes_to_wif import binascii import decimal @@ -28,10 +29,14 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): self.skip_if_no_wallet() def get_keys(self): + self.pub = [] + self.priv = [] node0, node1, node2 = self.nodes - add = [node1.getnewaddress() for _ in range(self.nkeys)] - self.pub = [node1.getaddressinfo(a)["pubkey"] for a in add] - self.priv = [node1.dumpprivkey(a) for a in add] + for _ in range(self.nkeys): + k = ECKey() + k.generate() + self.pub.append(k.get_pubkey().get_bytes().hex()) + self.priv.append(bytes_to_wif(k.get_bytes(), k.is_compressed)) self.final = node2.getnewaddress() def run_test(self): @@ -64,17 +69,20 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): pk_obj.compressed = False pk2 = binascii.hexlify(pk_obj.get_bytes()).decode() + node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) + wmulti0 = node0.get_wallet_rpc('wmulti0') + # Check all permutations of keys because order matters apparently for keys in itertools.permutations([pk0, pk1, pk2]): # Results should be the same as this legacy one legacy_addr = node0.createmultisig(2, keys, 'legacy')['address'] - assert_equal(legacy_addr, node0.addmultisigaddress(2, keys, '', 'legacy')['address']) + assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '', 'legacy')['address']) # Generate addresses with the segwit types. These should all make legacy addresses - assert_equal(legacy_addr, node0.createmultisig(2, keys, 'bech32')['address']) - assert_equal(legacy_addr, node0.createmultisig(2, keys, 'p2sh-segwit')['address']) - assert_equal(legacy_addr, node0.addmultisigaddress(2, keys, '', 'bech32')['address']) - assert_equal(legacy_addr, node0.addmultisigaddress(2, keys, '', 'p2sh-segwit')['address']) + assert_equal(legacy_addr, wmulti0.createmultisig(2, keys, 'bech32')['address']) + assert_equal(legacy_addr, wmulti0.createmultisig(2, keys, 'p2sh-segwit')['address']) + assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '', 'bech32')['address']) + assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '', 'p2sh-segwit')['address']) self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: @@ -89,6 +97,8 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): assert_equal(self.nodes[0].deriveaddresses(sorted_key_desc)[0], t['address']) def check_addmultisigaddress_errors(self): + if self.options.descriptors: + return self.log.info('Check that addmultisigaddress fails when the private keys are missing') addresses = [self.nodes[1].getnewaddress(address_type='legacy') for _ in range(2)] assert_raises_rpc_error(-5, 'no full public key for address', lambda: self.nodes[0].addmultisigaddress(nrequired=1, keys=addresses)) @@ -115,6 +125,15 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): def do_multisig(self): node0, node1, node2 = self.nodes + if 'wmulti' not in node1.listwallets(): + try: + node1.loadwallet('wmulti') + except JSONRPCException as e: + if e.error['code'] == -18 and 'Wallet wmulti not found' in e.error['message']: + node1.createwallet(wallet_name='wmulti', disable_private_keys=True) + else: + raise + wmulti = node1.get_wallet_rpc('wmulti') # Construct the expected descriptor desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub)) @@ -134,7 +153,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): assert madd[0:4] == "bcrt" # actually a bech32 address # compare against addmultisigaddress - msigw = node1.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) + msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type) maddw = msigw["address"] mredeemw = msigw["redeemScript"] assert_equal(desc, drop_origins(msigw['descriptor'])) @@ -194,6 +213,8 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): txinfo = node0.getrawtransaction(tx, True, blk) self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"])) + wmulti.unloadwallet() + if __name__ == '__main__': RpcCreateMultiSigTest().main() diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index ea8510f92b..51d136d26a 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -48,18 +48,23 @@ class PSBTTest(BitcoinTestFramework): disconnect_nodes(offline_node, 2) disconnect_nodes(mining_node, 0) + # Create watchonly on online_node + online_node.createwallet(wallet_name='wonline', disable_private_keys=True) + wonline = online_node.get_wallet_rpc('wonline') + w2 = online_node.get_wallet_rpc('') + # Mine a transaction that credits the offline address offline_addr = offline_node.getnewaddress(address_type="p2sh-segwit") - online_addr = online_node.getnewaddress(address_type="p2sh-segwit") - online_node.importaddress(offline_addr, "", False) + online_addr = w2.getnewaddress(address_type="p2sh-segwit") + wonline.importaddress(offline_addr, "", False) mining_node.sendtoaddress(address=offline_addr, amount=1.0) mining_node.generate(nblocks=1) self.sync_blocks([mining_node, online_node]) # Construct an unsigned PSBT on the online node (who doesn't know the output is Segwit, so will include a non-witness UTXO) - utxos = online_node.listunspent(addresses=[offline_addr]) - raw = online_node.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}]) - psbt = online_node.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"] + utxos = wonline.listunspent(addresses=[offline_addr]) + raw = wonline.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}]) + psbt = wonline.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"] assert "non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0] # Have the offline node sign the PSBT (which will update the UTXO to segwit) @@ -72,6 +77,8 @@ class PSBTTest(BitcoinTestFramework): self.sync_blocks([mining_node, online_node]) assert_equal(online_node.gettxout(txid,0)["confirmations"], 1) + wonline.unloadwallet() + # Reconnect connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) @@ -89,13 +96,23 @@ class PSBTTest(BitcoinTestFramework): final_tx = self.nodes[0].finalizepsbt(signed_tx)['hex'] self.nodes[0].sendrawtransaction(final_tx) - # Create p2sh, p2wpkh, and p2wsh addresses + # Get pubkeys pubkey0 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress())['pubkey'] pubkey1 = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['pubkey'] pubkey2 = self.nodes[2].getaddressinfo(self.nodes[2].getnewaddress())['pubkey'] - p2sh = self.nodes[1].addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "legacy")['address'] - p2wsh = self.nodes[1].addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "bech32")['address'] - p2sh_p2wsh = self.nodes[1].addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "p2sh-segwit")['address'] + + # Setup watchonly wallets + self.nodes[2].createwallet(wallet_name='wmulti', disable_private_keys=True) + wmulti = self.nodes[2].get_wallet_rpc('wmulti') + + # Create all the addresses + p2sh = wmulti.addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "legacy")['address'] + p2wsh = wmulti.addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "bech32")['address'] + p2sh_p2wsh = wmulti.addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "p2sh-segwit")['address'] + if not self.options.descriptors: + wmulti.importaddress(p2sh) + wmulti.importaddress(p2wsh) + wmulti.importaddress(p2sh_p2wsh) p2wpkh = self.nodes[1].getnewaddress("", "bech32") p2pkh = self.nodes[1].getnewaddress("", "legacy") p2sh_p2wpkh = self.nodes[1].getnewaddress("", "p2sh-segwit") @@ -146,11 +163,14 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Fee exceeds maximum configured by -maxtxfee", self.nodes[1].walletcreatefundedpsbt, [{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99}, 0, {"feeRate": 10}) # partially sign multisig things with node 1 - psbtx = self.nodes[1].walletcreatefundedpsbt([{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], {self.nodes[1].getnewaddress():29.99})['psbt'] + psbtx = wmulti.walletcreatefundedpsbt(inputs=[{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], outputs={self.nodes[1].getnewaddress():29.99}, options={'changeAddress': self.nodes[1].getrawchangeaddress()})['psbt'] walletprocesspsbt_out = self.nodes[1].walletprocesspsbt(psbtx) psbtx = walletprocesspsbt_out['psbt'] assert_equal(walletprocesspsbt_out['complete'], False) + # Unload wmulti, we don't need it anymore + wmulti.unloadwallet() + # partially sign with node 2. This should be complete and sendable walletprocesspsbt_out = self.nodes[2].walletprocesspsbt(psbtx) assert_equal(walletprocesspsbt_out['complete'], True) @@ -297,7 +317,7 @@ class PSBTTest(BitcoinTestFramework): # Signer tests for i, signer in enumerate(signers): - self.nodes[2].createwallet("wallet{}".format(i)) + self.nodes[2].createwallet(wallet_name="wallet{}".format(i)) wrpc = self.nodes[2].get_wallet_rpc("wallet{}".format(i)) for key in signer['privkeys']: wrpc.importprivkey(key) diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 912c0ca978..f2d6fba4a6 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -8,6 +8,8 @@ keys, and is trivially vulnerable to side channel attacks. Do not use for anything but tests.""" import random +from .address import byte_to_base58 + def modinv(a, n): """Compute the modular inverse of a modulo n @@ -384,3 +386,14 @@ class ECKey(): rb = r.to_bytes((r.bit_length() + 8) // 8, 'big') sb = s.to_bytes((s.bit_length() + 8) // 8, 'big') return b'\x30' + bytes([4 + len(rb) + len(sb), 2, len(rb)]) + rb + bytes([2, len(sb)]) + sb + +def bytes_to_wif(b, compressed=True): + if compressed: + b += b'\x01' + return byte_to_base58(b, 239) + +def generate_wif_key(): + # Makes a WIF privkey for imports + k = ECKey() + k.generate() + return bytes_to_wif(k.get_bytes(), k.is_compressed) diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 6aa73623e6..31cec66ee7 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -12,7 +12,10 @@ found in the mini-node branch of http://github.com/jgarzik/pynode. P2PConnection: A low-level connection object to a node's P2P interface P2PInterface: A high-level interface object for communicating to a node over P2P P2PDataStore: A p2p interface class that keeps a store of transactions and blocks - and can respond correctly to getdata and getheaders messages""" + and can respond correctly to getdata and getheaders messages +P2PTxInvStore: A p2p interface class that inherits from P2PDataStore, and keeps + a count of how many times each txid has been announced.""" + import asyncio from collections import defaultdict from io import BytesIO @@ -117,8 +120,9 @@ class P2PConnection(asyncio.Protocol): def is_connected(self): return self._transport is not None - def peer_connect(self, dstaddr, dstport, *, net): + def peer_connect(self, dstaddr, dstport, *, net, factor): assert not self.is_connected + self.factor = factor self.dstaddr = dstaddr self.dstport = dstport # The initial message to send after the connection was made: @@ -364,9 +368,12 @@ class P2PInterface(P2PConnection): # Connection helper methods + def wait_until(self, test_function, timeout): + wait_until(test_function, timeout=timeout, lock=mininode_lock, factor=self.factor) + def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.is_connected - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) # Message receiving helper methods @@ -377,14 +384,14 @@ class P2PInterface(P2PConnection): return False return self.last_message['tx'].tx.rehash() == txid - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_block(self, blockhash, timeout=60): def test_function(): assert self.is_connected return self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_header(self, blockhash, timeout=60): def test_function(): @@ -394,7 +401,7 @@ class P2PInterface(P2PConnection): return False return last_headers.headers[0].rehash() == int(blockhash, 16) - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_merkleblock(self, blockhash, timeout=60): def test_function(): @@ -404,7 +411,7 @@ class P2PInterface(P2PConnection): return False return last_filtered_block.merkleblock.header.rehash() == int(blockhash, 16) - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_getdata(self, hash_list, timeout=60): """Waits for a getdata message. @@ -418,7 +425,7 @@ class P2PInterface(P2PConnection): return False return [x.hash for x in last_data.inv] == hash_list - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_getheaders(self, timeout=60): """Waits for a getheaders message. @@ -432,7 +439,7 @@ class P2PInterface(P2PConnection): assert self.is_connected return self.last_message.get("getheaders") - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_inv(self, expected_inv, timeout=60): """Waits for an INV message and checks that the first inv object in the message was as expected.""" @@ -445,13 +452,13 @@ class P2PInterface(P2PConnection): self.last_message["inv"].inv[0].type == expected_inv[0].type and \ self.last_message["inv"].inv[0].hash == expected_inv[0].hash - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_verack(self, timeout=60): def test_function(): return self.message_count["verack"] - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) # Message sending helper functions @@ -467,7 +474,7 @@ class P2PInterface(P2PConnection): assert self.is_connected return self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) self.ping_counter += 1 @@ -583,7 +590,7 @@ class P2PDataStore(P2PInterface): self.send_message(msg_block(block=b)) else: self.send_message(msg_headers([CBlockHeader(block) for block in blocks])) - wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout, lock=mininode_lock) + self.wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout) if expect_disconnect: self.wait_for_disconnect(timeout=timeout) @@ -591,7 +598,7 @@ class P2PDataStore(P2PInterface): self.sync_with_ping(timeout=timeout) if success: - wait_until(lambda: node.getbestblockhash() == blocks[-1].hash, timeout=timeout) + self.wait_until(lambda: node.getbestblockhash() == blocks[-1].hash, timeout=timeout) else: assert node.getbestblockhash() != blocks[-1].hash @@ -627,3 +634,20 @@ class P2PDataStore(P2PInterface): # Check that none of the txs are now in the mempool for tx in txs: assert tx.hash not in raw_mempool, "{} tx found in mempool".format(tx.hash) + +class P2PTxInvStore(P2PInterface): + """A P2PInterface which stores a count of how many times each txid has been announced.""" + def __init__(self): + super().__init__() + self.tx_invs_received = defaultdict(int) + + def on_inv(self, message): + # Store how many times invs have been received for each tx. + for i in message.inv: + if i.type == MSG_TX: + # save txid + self.tx_invs_received[i.hash] += 1 + + def get_invs(self): + with mininode_lock: + return list(self.tx_invs_received.keys()) diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index e475ed8596..9102266456 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -8,6 +8,7 @@ This file is modified from python-bitcoinlib. """ import hashlib import struct +import unittest from .messages import ( CTransaction, @@ -708,3 +709,25 @@ def SegwitV0SignatureHash(script, txTo, inIdx, hashtype, amount): ss += struct.pack("<I", hashtype) return hash256(ss) + +class TestFrameworkScript(unittest.TestCase): + def test_bn2vch(self): + self.assertEqual(bn2vch(0), bytes([])) + self.assertEqual(bn2vch(1), bytes([0x01])) + self.assertEqual(bn2vch(-1), bytes([0x81])) + self.assertEqual(bn2vch(0x7F), bytes([0x7F])) + self.assertEqual(bn2vch(-0x7F), bytes([0xFF])) + self.assertEqual(bn2vch(0x80), bytes([0x80, 0x00])) + self.assertEqual(bn2vch(-0x80), bytes([0x80, 0x80])) + self.assertEqual(bn2vch(0xFF), bytes([0xFF, 0x00])) + self.assertEqual(bn2vch(-0xFF), bytes([0xFF, 0x80])) + self.assertEqual(bn2vch(0x100), bytes([0x00, 0x01])) + self.assertEqual(bn2vch(-0x100), bytes([0x00, 0x81])) + self.assertEqual(bn2vch(0x7FFF), bytes([0xFF, 0x7F])) + self.assertEqual(bn2vch(-0x8000), bytes([0x00, 0x80, 0x80])) + self.assertEqual(bn2vch(-0x7FFFFF), bytes([0xFF, 0xFF, 0xFF])) + self.assertEqual(bn2vch(0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x00])) + self.assertEqual(bn2vch(-0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x80])) + self.assertEqual(bn2vch(0xFFFFFFFF), bytes([0xFF, 0xFF, 0xFF, 0xFF, 0x00])) + self.assertEqual(bn2vch(123456789), bytes([0x15, 0xCD, 0x5B, 0x07])) + self.assertEqual(bn2vch(-54321), bytes([0x31, 0xD4, 0x80])) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index d4cf5f8896..11c96deefb 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -6,11 +6,12 @@ import configparser from enum import Enum -import logging import argparse +import logging import os import pdb import random +import re import shutil import subprocess import sys @@ -101,6 +102,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.bind_to_localhost_only = True self.set_test_params() self.parse_args() + self.rpc_timeout = int(self.rpc_timeout * self.options.factor) # optionally, increase timeout by a factor def main(self): """Main function. This should not be overridden by the subclass test scripts.""" @@ -165,6 +167,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="run nodes under the valgrind memory error detector: expect at least a ~10x slowdown, valgrind 3.14 or later required") parser.add_argument("--randomseed", type=int, help="set a random seed for deterministically reproducing a previous test run") + parser.add_argument("--descriptors", default=False, action="store_true", + help="Run test using a descriptor wallet") + parser.add_argument('--factor', type=float, default=1.0, help='adjust test timeouts by a factor') self.add_options(parser) self.options = parser.parse_args() @@ -183,10 +188,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.options.bitcoind = os.getenv("BITCOIND", default=config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"]) self.options.bitcoincli = os.getenv("BITCOINCLI", default=config["environment"]["BUILDDIR"] + '/src/bitcoin-cli' + config["environment"]["EXEEXT"]) + self.options.previous_releases_path = os.getenv("PREVIOUS_RELEASES_DIR") or os.getcwd() + "/releases" + os.environ['PATH'] = os.pathsep.join([ os.path.join(config['environment']['BUILDDIR'], 'src'), - os.path.join(config['environment']['BUILDDIR'], 'src', 'qt'), - os.environ['PATH'] + os.path.join(config['environment']['BUILDDIR'], 'src', 'qt'), os.environ['PATH'] ]) # Set up temp directory and start logging @@ -333,11 +339,23 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def setup_nodes(self): """Override this method to customize test node setup""" - extra_args = None + extra_args = [[]] * self.num_nodes + wallets = [[]] * self.num_nodes if hasattr(self, "extra_args"): extra_args = self.extra_args + wallets = [[x for x in eargs if x.startswith('-wallet=')] for eargs in extra_args] + extra_args = [x + ['-nowallet'] for x in extra_args] self.add_nodes(self.num_nodes, extra_args) self.start_nodes() + for i, n in enumerate(self.nodes): + n.extra_args.pop() + if '-wallet=0' in n.extra_args or '-nowallet' in n.extra_args or '-disablewallet' in n.extra_args or not self.is_wallet_compiled(): + continue + if '-wallet=' not in wallets[i] and not any([x.startswith('-wallet=') for x in wallets[i]]): + wallets[i].append('-wallet=') + for w in wallets[i]: + wallet_name = w.split('=', 1)[1] + n.createwallet(wallet_name=wallet_name, descriptors=self.options.descriptors) self.import_deterministic_coinbase_privkeys() if not self.setup_clean_chain: for n in self.nodes: @@ -374,6 +392,25 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): Should only be called once after the nodes have been specified in set_test_params().""" + def get_bin_from_version(version, bin_name, bin_default): + if not version: + return bin_default + return os.path.join( + self.options.previous_releases_path, + re.sub( + r'\.0$', + '', # remove trailing .0 for point releases + 'v{}.{}.{}.{}'.format( + (version % 100000000) // 1000000, + (version % 1000000) // 10000, + (version % 10000) // 100, + (version % 100) // 1, + ), + ), + 'bin', + bin_name, + ) + if self.bind_to_localhost_only: extra_confs = [["bind=127.0.0.1"]] * num_nodes else: @@ -383,9 +420,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if versions is None: versions = [None] * num_nodes if binary is None: - binary = [self.options.bitcoind] * num_nodes + binary = [get_bin_from_version(v, 'bitcoind', self.options.bitcoind) for v in versions] if binary_cli is None: - binary_cli = [self.options.bitcoincli] * num_nodes + binary_cli = [get_bin_from_version(v, 'bitcoin-cli', self.options.bitcoincli) for v in versions] assert_equal(len(extra_confs), num_nodes) assert_equal(len(extra_args), num_nodes) assert_equal(len(versions), num_nodes) @@ -398,6 +435,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): chain=self.chain, rpchost=rpchost, timewait=self.rpc_timeout, + factor=self.options.factor, bitcoind=binary[i], bitcoin_cli=binary_cli[i], version=versions[i], @@ -408,6 +446,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): use_cli=self.options.usecli, start_perf=self.options.perf, use_valgrind=self.options.valgrind, + descriptors=self.options.descriptors, )) def start_node(self, i, *args, **kwargs): @@ -543,10 +582,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args=['-disablewallet'], rpchost=None, timewait=self.rpc_timeout, + factor=self.options.factor, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, coverage_dir=None, cwd=self.options.tmpdir, + descriptors=self.options.descriptors, )) self.start_node(CACHE_NODE_ID) cache_node = self.nodes[CACHE_NODE_ID] @@ -624,6 +665,25 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_cli_compiled(): raise SkipTest("bitcoin-cli has not been compiled.") + def skip_if_no_previous_releases(self): + """Skip the running test if previous releases are not available.""" + if not self.has_previous_releases(): + raise SkipTest("previous releases not available or disabled") + + def has_previous_releases(self): + """Checks whether previous releases are present and enabled.""" + if os.getenv("TEST_PREVIOUS_RELEASES") == "false": + # disabled + return False + + if not os.path.isdir(self.options.previous_releases_path): + if os.getenv("TEST_PREVIOUS_RELEASES") == "true": + raise AssertionError("TEST_PREVIOUS_RELEASES=true but releases missing: {}".format( + self.options.previous_releases_path)) + # missing + return False + return True + def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" return self.config["components"].getboolean("ENABLE_CLI") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 507a0cff60..e6ec3c1b2d 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -22,6 +22,7 @@ import shlex import sys from .authproxy import JSONRPCException +from .descriptors import descsum_create from .util import ( MAX_NODES, append_config, @@ -61,7 +62,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, chain, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None): + def __init__(self, i, datadir, *, chain, rpchost, timewait, factor, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -79,6 +80,7 @@ class TestNode(): self.binary = bitcoind self.coverage_dir = coverage_dir self.cwd = cwd + self.descriptors = descriptors if extra_conf is not None: append_config(datadir, extra_conf) # Most callers will just need to add extra args to the standard list below. @@ -126,6 +128,7 @@ class TestNode(): self.perf_subprocesses = {} self.p2ps = [] + self.factor = factor AddressKeyPair = collections.namedtuple('AddressKeyPair', ['address', 'key']) PRIV_KEYS = [ @@ -170,10 +173,10 @@ class TestNode(): def __getattr__(self, name): """Dispatches any unrecognised messages to the RPC connection or a CLI instance.""" if self.use_cli: - return getattr(self.cli, name) + return getattr(RPCOverloadWrapper(self.cli, True, self.descriptors), name) else: assert self.rpc_connected and self.rpc is not None, self._node_msg("Error: no RPC connection") - return getattr(self.rpc, name) + return getattr(RPCOverloadWrapper(self.rpc, descriptors=self.descriptors), name) def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, **kwargs): """Start the node.""" @@ -265,11 +268,11 @@ class TestNode(): def get_wallet_rpc(self, wallet_name): if self.use_cli: - return self.cli("-rpcwallet={}".format(wallet_name)) + return RPCOverloadWrapper(self.cli("-rpcwallet={}".format(wallet_name)), True, self.descriptors) else: assert self.rpc_connected and self.rpc, self._node_msg("RPC not connected") wallet_path = "wallet/{}".format(urllib.parse.quote(wallet_name)) - return self.rpc / wallet_path + return RPCOverloadWrapper(self.rpc / wallet_path, descriptors=self.descriptors) def stop_node(self, expected_stderr='', wait=0): """Stop the node.""" @@ -322,13 +325,13 @@ class TestNode(): return True def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): - wait_until(self.is_node_stopped, timeout=timeout) + wait_until(self.is_node_stopped, timeout=timeout, factor=self.factor) @contextlib.contextmanager def assert_debug_log(self, expected_msgs, unexpected_msgs=None, timeout=2): if unexpected_msgs is None: unexpected_msgs = [] - time_end = time.time() + timeout + time_end = time.time() + timeout * self.factor debug_log = os.path.join(self.datadir, self.chain, 'debug.log') with open(debug_log, encoding='utf-8') as dl: dl.seek(0, 2) @@ -485,7 +488,7 @@ class TestNode(): if 'dstaddr' not in kwargs: kwargs['dstaddr'] = '127.0.0.1' - p2p_conn.peer_connect(**kwargs, net=self.chain)() + p2p_conn.peer_connect(**kwargs, net=self.chain, factor=self.factor)() self.p2ps.append(p2p_conn) if wait_for_verack: # Wait for the node to send us the version and verack @@ -595,3 +598,118 @@ class TestNodeCLI(): return json.loads(cli_stdout, parse_float=decimal.Decimal) except json.JSONDecodeError: return cli_stdout.rstrip("\n") + +class RPCOverloadWrapper(): + def __init__(self, rpc, cli=False, descriptors=False): + self.rpc = rpc + self.is_cli = cli + self.descriptors = descriptors + + def __getattr__(self, name): + return getattr(self.rpc, name) + + def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase=None, avoid_reuse=None, descriptors=None): + if self.is_cli: + if disable_private_keys is None: + disable_private_keys = 'null' + if blank is None: + blank = 'null' + if passphrase is None: + passphrase = '' + if avoid_reuse is None: + avoid_reuse = 'null' + if descriptors is None: + descriptors = self.descriptors + return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors) + + def importprivkey(self, privkey, label=None, rescan=None): + wallet_info = self.getwalletinfo() + if self.is_cli: + if label is None: + label = 'null' + if rescan is None: + rescan = 'null' + if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): + return self.__getattr__('importprivkey')(privkey, label, rescan) + desc = descsum_create('combo(' + privkey + ')') + req = [{ + 'desc': desc, + 'timestamp': 0 if rescan else 'now', + 'label': label if label else '' + }] + import_res = self.importdescriptors(req) + if not import_res[0]['success']: + raise JSONRPCException(import_res[0]['error']) + + def addmultisigaddress(self, nrequired, keys, label=None, address_type=None): + wallet_info = self.getwalletinfo() + if self.is_cli: + if label is None: + label = 'null' + if address_type is None: + address_type = 'null' + if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): + return self.__getattr__('addmultisigaddress')(nrequired, keys, label, address_type) + cms = self.createmultisig(nrequired, keys, address_type) + req = [{ + 'desc': cms['descriptor'], + 'timestamp': 0, + 'label': label if label else '' + }] + import_res = self.importdescriptors(req) + if not import_res[0]['success']: + raise JSONRPCException(import_res[0]['error']) + return cms + + def importpubkey(self, pubkey, label=None, rescan=None): + wallet_info = self.getwalletinfo() + if self.is_cli: + if label is None: + label = 'null' + if rescan is None: + rescan = 'null' + if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): + return self.__getattr__('importpubkey')(pubkey, label, rescan) + desc = descsum_create('combo(' + pubkey + ')') + req = [{ + 'desc': desc, + 'timestamp': 0 if rescan else 'now', + 'label': label if label else '' + }] + import_res = self.importdescriptors(req) + if not import_res[0]['success']: + raise JSONRPCException(import_res[0]['error']) + + def importaddress(self, address, label=None, rescan=None, p2sh=None): + wallet_info = self.getwalletinfo() + if self.is_cli: + if label is None: + label = 'null' + if rescan is None: + rescan = 'null' + if p2sh is None: + p2sh = 'null' + if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): + return self.__getattr__('importaddress')(address, label, rescan, p2sh) + is_hex = False + try: + int(address ,16) + is_hex = True + desc = descsum_create('raw(' + address + ')') + except: + desc = descsum_create('addr(' + address + ')') + reqs = [{ + 'desc': desc, + 'timestamp': 0 if rescan else 'now', + 'label': label if label else '' + }] + if is_hex and p2sh: + reqs.append({ + 'desc': descsum_create('p2sh(raw(' + address + '))'), + 'timestamp': 0 if rescan else 'now', + 'label': label if label else '' + }) + import_res = self.importdescriptors(reqs) + for res in import_res: + if not res['success']: + raise JSONRPCException(res['error']) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 64e1aa3bbc..20ab9ee464 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -208,9 +208,10 @@ def str_to_b64str(string): def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) -def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None): +def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, factor=1.0): if attempts == float('inf') and timeout == float('inf'): timeout = 60 + timeout = timeout * factor attempt = 0 time_end = time.time() + timeout @@ -265,7 +266,7 @@ def get_rpc_proxy(url, node_number, *, timeout=None, coveragedir=None): """ proxy_kwargs = {} if timeout is not None: - proxy_kwargs['timeout'] = timeout + proxy_kwargs['timeout'] = int(timeout) proxy = AuthServiceProxy(url, **proxy_kwargs) proxy.url = url # store URL on proxy for info @@ -326,6 +327,13 @@ def initialize_datadir(dirname, n, chain): os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) return datadir +def adjust_bitcoin_conf_for_pre_17(conf_file): + with open(conf_file,'r', encoding='utf8') as conf: + conf_data = conf.read() + with open(conf_file, 'w', encoding='utf8') as conf: + conf_data_changed = conf_data.replace('[regtest]', '') + conf.write(conf_data_changed) + def get_datadir_path(dirname, n): return os.path.join(dirname, "node" + str(n)) diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index eb537015fb..1b6686ff45 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -13,6 +13,10 @@ from test_framework.address import ( script_to_p2sh_p2wsh, script_to_p2wsh, ) +from test_framework.key import ( + bytes_to_wif, + ECKey, +) from test_framework.script import ( CScript, OP_0, @@ -66,6 +70,25 @@ def get_key(node): p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(), p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey)) +def get_generate_key(): + """Generate a fresh key + + Returns a named tuple of privkey, pubkey and all address and scripts.""" + eckey = ECKey() + eckey.generate() + privkey = bytes_to_wif(eckey.get_bytes()) + pubkey = eckey.get_pubkey().get_bytes().hex() + pkh = hash160(hex_str_to_bytes(pubkey)) + return Key(privkey=privkey, + pubkey=pubkey, + p2pkh_script=CScript([OP_DUP, OP_HASH160, pkh, OP_EQUALVERIFY, OP_CHECKSIG]).hex(), + p2pkh_addr=key_to_p2pkh(pubkey), + p2wpkh_script=CScript([OP_0, pkh]).hex(), + p2wpkh_addr=key_to_p2wpkh(pubkey), + p2sh_p2wpkh_script=CScript([OP_HASH160, hash160(CScript([OP_0, pkh])), OP_EQUAL]).hex(), + p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(), + p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey)) + def get_multisig(node): """Generate a fresh 2-of-3 multisig on node diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 4f902717c3..45f591ba4a 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -24,6 +24,7 @@ import sys import tempfile import re import logging +import unittest # Formatting. Default colors to empty strings. BOLD, GREEN, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") @@ -65,6 +66,10 @@ if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 +TEST_FRAMEWORK_MODULES = [ + "script", +] + EXTENDED_SCRIPTS = [ # These tests are not run by default. # Longest test should go first, to favor running tests in parallel @@ -76,6 +81,7 @@ BASE_SCRIPTS = [ # Scripts that are run by default. # Longest test should go first, to favor running tests in parallel 'wallet_hd.py', + 'wallet_hd.py --descriptors', 'wallet_backup.py', # vv Tests less than 5m vv 'mining_getblocktemplate_longpoll.py', @@ -86,10 +92,13 @@ BASE_SCRIPTS = [ 'feature_segwit.py', # vv Tests less than 2m vv 'wallet_basic.py', + 'wallet_basic.py --descriptors', 'wallet_labels.py', + 'wallet_labels.py --descriptors', 'p2p_segwit.py', 'p2p_timeouts.py', 'p2p_tx_download.py', + 'mempool_updatefromblock.py', 'wallet_dump.py', 'wallet_listtransactions.py', # vv Tests less than 60s vv @@ -109,6 +118,7 @@ BASE_SCRIPTS = [ 'feature_abortnode.py', # vv Tests less than 30s vv 'wallet_keypool_topup.py', + 'wallet_keypool_topup.py --descriptors', 'feature_fee_estimation.py', 'interface_zmq.py', 'interface_bitcoin_cli.py', @@ -122,6 +132,7 @@ BASE_SCRIPTS = [ 'interface_rest.py', 'mempool_spend_coinbase.py', 'wallet_avoidreuse.py', + 'wallet_avoidreuse.py --descriptors', 'mempool_reorg.py', 'mempool_persist.py', 'wallet_multiwallet.py', @@ -134,6 +145,7 @@ BASE_SCRIPTS = [ 'interface_http.py', 'interface_rpc.py', 'rpc_psbt.py', + 'rpc_psbt.py --descriptors', 'rpc_users.py', 'rpc_whitelist.py', 'feature_proxy.py', @@ -147,6 +159,8 @@ BASE_SCRIPTS = [ 'p2p_addr_relay.py', 'rpc_net.py', 'wallet_keypool.py', + 'wallet_keypool.py --descriptors', + 'wallet_descriptor.py', 'p2p_mempool.py', 'p2p_filter.py', 'rpc_setban.py', @@ -168,6 +182,7 @@ BASE_SCRIPTS = [ 'mempool_packages.py', 'mempool_package_onemore.py', 'rpc_createmultisig.py', + 'rpc_createmultisig.py --descriptors', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', 'wallet_importprunedfunds.py', @@ -180,6 +195,8 @@ BASE_SCRIPTS = [ 'mempool_expiry.py', 'wallet_import_rescan.py', 'wallet_import_with_label.py', + 'wallet_importdescriptors.py', + 'wallet_upgradewallet.py', 'rpc_bind.py --ipv4', 'rpc_bind.py --ipv6', 'rpc_bind.py --nonloopback', @@ -190,6 +207,7 @@ BASE_SCRIPTS = [ 'wallet_listsinceblock.py', 'p2p_leak.py', 'wallet_encryption.py', + 'wallet_encryption.py --descriptors', 'feature_dersig.py', 'feature_cltv.py', 'rpc_uptime.py', @@ -209,6 +227,7 @@ BASE_SCRIPTS = [ 'p2p_unrequested_blocks.py', 'feature_includeconf.py', 'feature_asmap.py', + 'mempool_unbroadcast.py', 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', 'rpc_scantxoutset.py', @@ -223,7 +242,6 @@ BASE_SCRIPTS = [ 'rpc_help.py', 'feature_help.py', 'feature_shutdown.py', - 'framework_test_script.py', # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] @@ -386,6 +404,16 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= if os.path.isdir(cache_dir): print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir)) + # Test Framework Tests + print("Running Unit Tests for Test Framework Modules") + test_framework_tests = unittest.TestSuite() + for module in TEST_FRAMEWORK_MODULES: + test_framework_tests.addTest(unittest.TestLoader().loadTestsFromName("test_framework.{}".format(module))) + result = unittest.TextTestRunner(verbosity=1, failfast=True).run(test_framework_tests) + if not result.wasSuccessful(): + logging.debug("Early exiting after failure in TestFramework unit tests") + sys.exit(False) + tests_dir = src_dir + '/test/functional/' flags = ['--cachedir={}'.format(cache_dir)] + args @@ -609,7 +637,7 @@ class TestResult(): def check_script_prefixes(): """Check that test scripts start with one of the allowed name prefixes.""" - good_prefixes_re = re.compile("^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool|framework_test)_") + good_prefixes_re = re.compile("^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_") bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None] if bad_script_names: diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index b3d496dd51..039ce7daee 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -15,6 +15,7 @@ from test_framework.util import assert_equal BUFFER_SIZE = 16 * 1024 + class ToolWalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -48,7 +49,7 @@ class ToolWalletTest(BitcoinTestFramework): h = hashlib.sha1() mv = memoryview(bytearray(BUFFER_SIZE)) with open(self.wallet_path, 'rb', buffering=0) as f: - for n in iter(lambda : f.readinto(mv), 0): + for n in iter(lambda: f.readinto(mv), 0): h.update(mv[:n]) return h.hexdigest() @@ -69,7 +70,12 @@ class ToolWalletTest(BitcoinTestFramework): self.assert_raises_tool_error('Invalid command: help', 'help') self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') - self.assert_raises_tool_error('Error loading wallet.dat. Is wallet being used by other process?', '-wallet=wallet.dat', 'info') + self.assert_raises_tool_error( + 'Error initializing wallet database environment "{}"!\nError loading wallet.dat. Is wallet being used by other process?' + .format(os.path.join(self.nodes[0].datadir, self.chain, 'wallets')), + '-wallet=wallet.dat', + 'info', + ) self.assert_raises_tool_error('Error: no wallet file at nonexistent.dat', '-wallet=nonexistent.dat', 'info') def test_tool_wallet_info(self): @@ -84,7 +90,7 @@ class ToolWalletTest(BitcoinTestFramework): # # self.log.debug('Setting wallet file permissions to 400 (read-only)') # os.chmod(self.wallet_path, stat.S_IRUSR) - # assert(self.wallet_permissions() in ['400', '666']) # Sanity check. 666 because Appveyor. + # assert self.wallet_permissions() in ['400', '666'] # Sanity check. 666 because Appveyor. # shasum_before = self.wallet_shasum() timestamp_before = self.wallet_timestamp() self.log.debug('Wallet file timestamp before calling info: {}'.format(timestamp_before)) @@ -103,7 +109,7 @@ class ToolWalletTest(BitcoinTestFramework): self.log_wallet_timestamp_comparison(timestamp_before, timestamp_after) self.log.debug('Setting wallet file permissions back to 600 (read/write)') os.chmod(self.wallet_path, stat.S_IRUSR | stat.S_IWUSR) - assert(self.wallet_permissions() in ['600', '666']) # Sanity check. 666 because Appveyor. + assert self.wallet_permissions() in ['600', '666'] # Sanity check. 666 because Appveyor. # # TODO: Wallet tool info should not write to the wallet file. # The following lines should be uncommented and the tests still succeed: diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 78a51a1d5f..780cce9d02 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -133,7 +133,7 @@ class AvoidReuseTest(BitcoinTestFramework): tempwallet = ".wallet_avoidreuse.py_test_immutable_wallet.dat" # Create a wallet with disable_private_keys set; this should work - self.nodes[1].createwallet(tempwallet, True) + self.nodes[1].createwallet(wallet_name=tempwallet, disable_private_keys=True) w = self.nodes[1].get_wallet_rpc(tempwallet) # Attempt to unset the disable_private_keys flag; this should not work @@ -249,43 +249,44 @@ class AvoidReuseTest(BitcoinTestFramework): # getbalances should show no used, 5 btc trusted assert_balances(self.nodes[1], mine={"used": 0, "trusted": 5}) - # For the second send, we transmute it to a related single-key address - # to make sure it's also detected as re-use - fund_spk = self.nodes[0].getaddressinfo(fundaddr)["scriptPubKey"] - fund_decoded = self.nodes[0].decodescript(fund_spk) - if second_addr_type == "p2sh-segwit": - new_fundaddr = fund_decoded["segwit"]["p2sh-segwit"] - elif second_addr_type == "bech32": - new_fundaddr = fund_decoded["segwit"]["addresses"][0] - else: - new_fundaddr = fundaddr - assert_equal(second_addr_type, "legacy") - - self.nodes[0].sendtoaddress(new_fundaddr, 10) - self.nodes[0].generate(1) - self.sync_all() - - # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10) - assert_unspent(self.nodes[1], total_count=2, total_sum=15, reused_count=1, reused_sum=10) - # getbalances should show 10 used, 5 btc trusted - assert_balances(self.nodes[1], mine={"used": 10, "trusted": 5}) - - # node 1 should now have a balance of 5 (no dirty) or 15 (including dirty) - assert_approx(self.nodes[1].getbalance(), 5, 0.001) - assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 15, 0.001) - - assert_raises_rpc_error(-6, "Insufficient funds", self.nodes[1].sendtoaddress, retaddr, 10) - - self.nodes[1].sendtoaddress(retaddr, 4) - - # listunspent should show 2 total outputs (1, 10 btc), one unused (1), one reused (10) - assert_unspent(self.nodes[1], total_count=2, total_sum=11, reused_count=1, reused_sum=10) - # getbalances should show 10 used, 1 btc trusted - assert_balances(self.nodes[1], mine={"used": 10, "trusted": 1}) - - # node 1 should now have about 1 btc left (no dirty) and 11 (including dirty) - assert_approx(self.nodes[1].getbalance(), 1, 0.001) - assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 11, 0.001) + if not self.options.descriptors: + # For the second send, we transmute it to a related single-key address + # to make sure it's also detected as re-use + fund_spk = self.nodes[0].getaddressinfo(fundaddr)["scriptPubKey"] + fund_decoded = self.nodes[0].decodescript(fund_spk) + if second_addr_type == "p2sh-segwit": + new_fundaddr = fund_decoded["segwit"]["p2sh-segwit"] + elif second_addr_type == "bech32": + new_fundaddr = fund_decoded["segwit"]["addresses"][0] + else: + new_fundaddr = fundaddr + assert_equal(second_addr_type, "legacy") + + self.nodes[0].sendtoaddress(new_fundaddr, 10) + self.nodes[0].generate(1) + self.sync_all() + + # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10) + assert_unspent(self.nodes[1], total_count=2, total_sum=15, reused_count=1, reused_sum=10) + # getbalances should show 10 used, 5 btc trusted + assert_balances(self.nodes[1], mine={"used": 10, "trusted": 5}) + + # node 1 should now have a balance of 5 (no dirty) or 15 (including dirty) + assert_approx(self.nodes[1].getbalance(), 5, 0.001) + assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 15, 0.001) + + assert_raises_rpc_error(-6, "Insufficient funds", self.nodes[1].sendtoaddress, retaddr, 10) + + self.nodes[1].sendtoaddress(retaddr, 4) + + # listunspent should show 2 total outputs (1, 10 btc), one unused (1), one reused (10) + assert_unspent(self.nodes[1], total_count=2, total_sum=11, reused_count=1, reused_sum=10) + # getbalances should show 10 used, 1 btc trusted + assert_balances(self.nodes[1], mine={"used": 10, "trusted": 1}) + + # node 1 should now have about 1 btc left (no dirty) and 11 (including dirty) + assert_approx(self.nodes[1].getbalance(), 1, 0.001) + assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 11, 0.001) def test_getbalances_used(self): ''' diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 15746d312c..2dddbf2cf3 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -49,6 +49,7 @@ class WalletTest(BitcoinTestFramework): return self.nodes[0].decoderawtransaction(txn)['vsize'] def run_test(self): + # Check that there's no UTXO on none of the nodes assert_equal(len(self.nodes[0].listunspent()), 0) assert_equal(len(self.nodes[1].listunspent()), 0) @@ -219,7 +220,7 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), node_2_bal) node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) - self.start_node(3) + self.start_node(3, self.nodes[3].extra_args) connect_nodes(self.nodes[0], 3) self.sync_all() @@ -315,57 +316,59 @@ class WalletTest(BitcoinTestFramework): # This will raise an exception since generate does not accept a string assert_raises_rpc_error(-1, "not an integer", self.nodes[0].generate, "2") - # This will raise an exception for the invalid private key format - assert_raises_rpc_error(-5, "Invalid private key encoding", self.nodes[0].importprivkey, "invalid") + if not self.options.descriptors: - # This will raise an exception for importing an address with the PS2H flag - temp_address = self.nodes[1].getnewaddress("", "p2sh-segwit") - assert_raises_rpc_error(-5, "Cannot use the p2sh flag with an address - use a script instead", self.nodes[0].importaddress, temp_address, "label", False, True) + # This will raise an exception for the invalid private key format + assert_raises_rpc_error(-5, "Invalid private key encoding", self.nodes[0].importprivkey, "invalid") - # This will raise an exception for attempting to dump the private key of an address you do not own - assert_raises_rpc_error(-3, "Address does not refer to a key", self.nodes[0].dumpprivkey, temp_address) + # This will raise an exception for importing an address with the PS2H flag + temp_address = self.nodes[1].getnewaddress("", "p2sh-segwit") + assert_raises_rpc_error(-5, "Cannot use the p2sh flag with an address - use a script instead", self.nodes[0].importaddress, temp_address, "label", False, True) - # This will raise an exception for attempting to get the private key of an invalid Bitcoin address - assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].dumpprivkey, "invalid") + # This will raise an exception for attempting to dump the private key of an address you do not own + assert_raises_rpc_error(-3, "Address does not refer to a key", self.nodes[0].dumpprivkey, temp_address) - # This will raise an exception for attempting to set a label for an invalid Bitcoin address - assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].setlabel, "invalid address", "label") + # This will raise an exception for attempting to get the private key of an invalid Bitcoin address + assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].dumpprivkey, "invalid") - # This will raise an exception for importing an invalid address - assert_raises_rpc_error(-5, "Invalid Bitcoin address or script", self.nodes[0].importaddress, "invalid") + # This will raise an exception for attempting to set a label for an invalid Bitcoin address + assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].setlabel, "invalid address", "label") - # This will raise an exception for attempting to import a pubkey that isn't in hex - assert_raises_rpc_error(-5, "Pubkey must be a hex string", self.nodes[0].importpubkey, "not hex") + # This will raise an exception for importing an invalid address + assert_raises_rpc_error(-5, "Invalid Bitcoin address or script", self.nodes[0].importaddress, "invalid") - # This will raise an exception for importing an invalid pubkey - assert_raises_rpc_error(-5, "Pubkey is not a valid public key", self.nodes[0].importpubkey, "5361746f736869204e616b616d6f746f") + # This will raise an exception for attempting to import a pubkey that isn't in hex + assert_raises_rpc_error(-5, "Pubkey must be a hex string", self.nodes[0].importpubkey, "not hex") - # Import address and private key to check correct behavior of spendable unspents - # 1. Send some coins to generate new UTXO - address_to_import = self.nodes[2].getnewaddress() - txid = self.nodes[0].sendtoaddress(address_to_import, 1) - self.nodes[0].generate(1) - self.sync_all(self.nodes[0:3]) + # This will raise an exception for importing an invalid pubkey + assert_raises_rpc_error(-5, "Pubkey is not a valid public key", self.nodes[0].importpubkey, "5361746f736869204e616b616d6f746f") + + # Import address and private key to check correct behavior of spendable unspents + # 1. Send some coins to generate new UTXO + address_to_import = self.nodes[2].getnewaddress() + txid = self.nodes[0].sendtoaddress(address_to_import, 1) + self.nodes[0].generate(1) + self.sync_all(self.nodes[0:3]) - # 2. Import address from node2 to node1 - self.nodes[1].importaddress(address_to_import) + # 2. Import address from node2 to node1 + self.nodes[1].importaddress(address_to_import) - # 3. Validate that the imported address is watch-only on node1 - assert self.nodes[1].getaddressinfo(address_to_import)["iswatchonly"] + # 3. Validate that the imported address is watch-only on node1 + assert self.nodes[1].getaddressinfo(address_to_import)["iswatchonly"] - # 4. Check that the unspents after import are not spendable - assert_array_result(self.nodes[1].listunspent(), - {"address": address_to_import}, - {"spendable": False}) + # 4. Check that the unspents after import are not spendable + assert_array_result(self.nodes[1].listunspent(), + {"address": address_to_import}, + {"spendable": False}) - # 5. Import private key of the previously imported address on node1 - priv_key = self.nodes[2].dumpprivkey(address_to_import) - self.nodes[1].importprivkey(priv_key) + # 5. Import private key of the previously imported address on node1 + priv_key = self.nodes[2].dumpprivkey(address_to_import) + self.nodes[1].importprivkey(priv_key) - # 6. Check that the unspents are now spendable on node1 - assert_array_result(self.nodes[1].listunspent(), - {"address": address_to_import}, - {"spendable": True}) + # 6. Check that the unspents are now spendable on node1 + assert_array_result(self.nodes[1].listunspent(), + {"address": address_to_import}, + {"spendable": True}) # Mine a block from node0 to an address from node1 coinbase_addr = self.nodes[1].getnewaddress() @@ -460,7 +463,8 @@ class WalletTest(BitcoinTestFramework): # Try with walletrejectlongchains # Double chain limit but require combining inputs, so we pass SelectCoinsMinConf self.stop_node(0) - self.start_node(0, extra_args=["-walletrejectlongchains", "-limitancestorcount=" + str(2 * chainlimit)]) + extra_args = ["-walletrejectlongchains", "-limitancestorcount=" + str(2 * chainlimit)] + self.start_node(0, extra_args=extra_args) # wait for loadmempool timeout = 10 diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index c09ca8854f..27197e3b6d 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -23,7 +23,6 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, - connect_nodes, hex_str_to_bytes, ) @@ -37,6 +36,7 @@ NORMAL = 0.00100000 HIGH = 0.00500000 TOO_HIGH = 1.00000000 + class BumpFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -55,9 +55,6 @@ class BumpFeeTest(BitcoinTestFramework): self.nodes[1].encryptwallet(WALLET_PASSPHRASE) self.nodes[1].walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) - connect_nodes(self.nodes[0], 1) - self.sync_all() - peer_node, rbf_node = self.nodes rbf_node_address = rbf_node.getnewaddress() @@ -94,7 +91,6 @@ class BumpFeeTest(BitcoinTestFramework): # These tests wipe out a number of utxos that are expected in other tests test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) test_no_more_inputs_fails(self, rbf_node, dest_address) - self.log.info("Success") def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): @@ -124,6 +120,7 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): assert_equal(oldwtx["replaced_by_txid"], bumped_tx["txid"]) assert_equal(bumpedwtx["replaces_txid"], rbfid) + def test_feerate_args(self, rbf_node, peer_node, dest_address): self.log.info('Test fee_rate args') rbfid = spend_one_input(rbf_node, dest_address) @@ -137,7 +134,7 @@ def test_feerate_args(self, rbf_node, peer_node, dest_address): # Bumping to just above minrelay should fail to increase total fee enough, at least assert_raises_rpc_error(-8, "Insufficient total fee", rbf_node.bumpfee, rbfid, {"fee_rate": INSUFFICIENT}) - assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate":-1}) + assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate": -1}) assert_raises_rpc_error(-4, "is too high (cannot be higher than", rbf_node.bumpfee, rbfid, {"fee_rate": TOO_HIGH}) @@ -209,6 +206,7 @@ def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_ad rbf_node.sendrawtransaction(tx["hex"]) assert_raises_rpc_error(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): self.log.info('Testing small output with feerate bump succeeds') @@ -249,6 +247,7 @@ def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): rbf_node.generatetoaddress(1, rbf_node.getnewaddress()) assert_equal(rbf_node.gettransaction(rbfid)["confirmations"], 1) + def test_dust_to_fee(self, rbf_node, dest_address): self.log.info('Test that bumped output that is dust is dropped to fee') rbfid = spend_one_input(rbf_node, dest_address) @@ -301,10 +300,11 @@ def test_maxtxfee_fails(self, rbf_node, dest_address): self.restart_node(1, ['-maxtxfee=0.000025'] + self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) rbfid = spend_one_input(rbf_node, dest_address) - assert_raises_rpc_error(-4, "Unable to create transaction: Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) + assert_raises_rpc_error(-4, "Unable to create transaction. Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) self.restart_node(1, self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): self.log.info('Test that PSBT is returned for bumpfee in watchonly wallets') priv_rec_desc = "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#rweraev0" @@ -339,12 +339,11 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): result = watcher.importmulti([{ "desc": pub_rec_desc, "timestamp": 0, - "range": [0,10], + "range": [0, 10], "internal": False, "keypool": True, "watchonly": True - }, - { + }, { "desc": pub_change_desc, "timestamp": 0, "range": [0, 10], @@ -361,7 +360,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): self.sync_all() # Create single-input PSBT for transaction to be bumped - psbt = watcher.walletcreatefundedpsbt([], {dest_address:0.0005}, 0, {"feeRate": 0.00001}, True)['psbt'] + psbt = watcher.walletcreatefundedpsbt([], {dest_address: 0.0005}, 0, {"feeRate": 0.00001}, True)['psbt'] psbt_signed = signer.walletprocesspsbt(psbt=psbt, sign=True, sighashtype="ALL", bip32derivs=True) psbt_final = watcher.finalizepsbt(psbt_signed["psbt"]) original_txid = watcher.sendrawtransaction(psbt_final["hex"]) @@ -387,6 +386,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): rbf_node.unloadwallet("watcher") rbf_node.unloadwallet("signer") + def test_rebumping(self, rbf_node, dest_address): self.log.info('Test that re-bumping the original tx fails, but bumping successor works') rbfid = spend_one_input(rbf_node, dest_address) @@ -461,6 +461,7 @@ def test_locked_wallet_fails(self, rbf_node, dest_address): rbf_node.bumpfee, rbfid) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + def test_change_script_match(self, rbf_node, dest_address): self.log.info('Test that the same change addresses is used for the replacement transaction when possible') @@ -480,6 +481,7 @@ def test_change_script_match(self, rbf_node, dest_address): bumped_rate_tx = rbf_node.bumpfee(bumped_total_tx["txid"]) assert_equal(change_addresses, get_change_address(bumped_rate_tx['txid'])) + def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): tx_input = dict( sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in node.listunspent() if u["amount"] == Decimal("0.00100000"))) @@ -491,6 +493,7 @@ def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): txid = node.sendrawtransaction(signedtx["hex"]) return txid + def submit_block_with_tx(node, tx): ctx = CTransaction() ctx.deserialize(io.BytesIO(hex_str_to_bytes(tx))) @@ -507,13 +510,14 @@ def submit_block_with_tx(node, tx): node.submitblock(block.serialize().hex()) return block + def test_no_more_inputs_fails(self, rbf_node, dest_address): self.log.info('Test that bumpfee fails when there are no available confirmed outputs') # feerate rbf requires confirmed outputs when change output doesn't exist or is insufficient rbf_node.generatetoaddress(1, dest_address) # spend all funds, no change output rbfid = rbf_node.sendtoaddress(rbf_node.getnewaddress(), rbf_node.getbalance(), "", "", True) - assert_raises_rpc_error(-4, "Unable to create transaction: Insufficient funds", rbf_node.bumpfee, rbfid) + assert_raises_rpc_error(-4, "Unable to create transaction. Insufficient funds", rbf_node.bumpfee, rbfid) if __name__ == "__main__": diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py new file mode 100755 index 0000000000..289ccf43ec --- /dev/null +++ b/test/functional/wallet_descriptor.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test descriptor wallet function.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error +) + + +class WalletDescriptorTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [['-keypool=100']] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + # Make a descriptor wallet + self.log.info("Making a descriptor wallet") + self.nodes[0].createwallet(wallet_name="desc1", descriptors=True) + self.nodes[0].unloadwallet("") + + # A descriptor wallet should have 100 addresses * 3 types = 300 keys + self.log.info("Checking wallet info") + wallet_info = self.nodes[0].getwalletinfo() + assert_equal(wallet_info['keypoolsize'], 300) + assert_equal(wallet_info['keypoolsize_hd_internal'], 300) + assert 'keypoololdest' not in wallet_info + + # Check that getnewaddress works + self.log.info("Test that getnewaddress and getrawchangeaddress work") + addr = self.nodes[0].getnewaddress("", "legacy") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('pkh(') + assert_equal(addr_info['hdkeypath'], 'm/44\'/1\'/0\'/0/0') + + addr = self.nodes[0].getnewaddress("", "p2sh-segwit") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('sh(wpkh(') + assert_equal(addr_info['hdkeypath'], 'm/49\'/1\'/0\'/0/0') + + addr = self.nodes[0].getnewaddress("", "bech32") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('wpkh(') + assert_equal(addr_info['hdkeypath'], 'm/84\'/1\'/0\'/0/0') + + # Check that getrawchangeaddress works + addr = self.nodes[0].getrawchangeaddress("legacy") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('pkh(') + assert_equal(addr_info['hdkeypath'], 'm/44\'/1\'/0\'/1/0') + + addr = self.nodes[0].getrawchangeaddress("p2sh-segwit") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('sh(wpkh(') + assert_equal(addr_info['hdkeypath'], 'm/49\'/1\'/0\'/1/0') + + addr = self.nodes[0].getrawchangeaddress("bech32") + addr_info = self.nodes[0].getaddressinfo(addr) + assert addr_info['desc'].startswith('wpkh(') + assert_equal(addr_info['hdkeypath'], 'm/84\'/1\'/0\'/1/0') + + # Make a wallet to receive coins at + self.nodes[0].createwallet(wallet_name="desc2", descriptors=True) + recv_wrpc = self.nodes[0].get_wallet_rpc("desc2") + send_wrpc = self.nodes[0].get_wallet_rpc("desc1") + + # Generate some coins + send_wrpc.generatetoaddress(101, send_wrpc.getnewaddress()) + + # Make transactions + self.log.info("Test sending and receiving") + addr = recv_wrpc.getnewaddress() + send_wrpc.sendtoaddress(addr, 10) + + # Make sure things are disabled + self.log.info("Test disabled RPCs") + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.importprivkey, "cVpF924EspNh8KjYsfhgY96mmxvT6DgdWiTYMtMjuM74hJaU5psW") + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.importpubkey, send_wrpc.getaddressinfo(send_wrpc.getnewaddress())) + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.importaddress, recv_wrpc.getnewaddress()) + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.importmulti, []) + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.addmultisigaddress, 1, [recv_wrpc.getnewaddress()]) + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.dumpprivkey, recv_wrpc.getnewaddress()) + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.dumpwallet, 'wallet.dump') + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.importwallet, 'wallet.dump') + assert_raises_rpc_error(-4, "This type of wallet does not support this command", recv_wrpc.rpc.sethdseed) + + self.log.info("Test encryption") + # Get the master fingerprint before encrypt + info1 = send_wrpc.getaddressinfo(send_wrpc.getnewaddress()) + + # Encrypt wallet 0 + send_wrpc.encryptwallet('pass') + send_wrpc.walletpassphrase('pass', 10) + addr = send_wrpc.getnewaddress() + info2 = send_wrpc.getaddressinfo(addr) + assert info1['hdmasterfingerprint'] != info2['hdmasterfingerprint'] + send_wrpc.walletlock() + assert 'hdmasterfingerprint' in send_wrpc.getaddressinfo(send_wrpc.getnewaddress()) + info3 = send_wrpc.getaddressinfo(addr) + assert_equal(info2['desc'], info3['desc']) + + self.log.info("Test that getnewaddress still works after keypool is exhausted in an encrypted wallet") + for i in range(0, 500): + send_wrpc.getnewaddress() + + self.log.info("Test that unlock is needed when deriving only hardened keys in an encrypted wallet") + send_wrpc.walletpassphrase('pass', 10) + send_wrpc.importdescriptors([{ + "desc": "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/*h)#y4dfsj7n", + "timestamp": "now", + "range": [0,10], + "active": True + }]) + send_wrpc.walletlock() + # Exhaust keypool of 100 + for i in range(0, 100): + send_wrpc.getnewaddress(address_type='bech32') + # This should now error + assert_raises_rpc_error(-12, "Keypool ran out, please call keypoolrefill first", send_wrpc.getnewaddress, '', 'bech32') + + self.log.info("Test born encrypted wallets") + self.nodes[0].createwallet('desc_enc', False, False, 'pass', False, True) + enc_rpc = self.nodes[0].get_wallet_rpc('desc_enc') + enc_rpc.getnewaddress() # Makes sure that we can get a new address from a born encrypted wallet + + self.log.info("Test blank descriptor wallets") + self.nodes[0].createwallet(wallet_name='desc_blank', blank=True, descriptors=True) + blank_rpc = self.nodes[0].get_wallet_rpc('desc_blank') + assert_raises_rpc_error(-4, 'This wallet has no available keys', blank_rpc.getnewaddress) + + self.log.info("Test descriptor wallet with disabled private keys") + self.nodes[0].createwallet(wallet_name='desc_no_priv', disable_private_keys=True, descriptors=True) + nopriv_rpc = self.nodes[0].get_wallet_rpc('desc_no_priv') + assert_raises_rpc_error(-4, 'This wallet has no available keys', nopriv_rpc.getnewaddress) + +if __name__ == '__main__': + WalletDescriptorTest().main () diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index bc7e3cca59..6cd82ad250 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -8,7 +8,6 @@ import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - assert_equal, assert_raises_rpc_error, assert_greater_than, assert_greater_than_or_equal, @@ -27,10 +26,10 @@ class WalletEncryptionTest(BitcoinTestFramework): passphrase2 = "SecondWalletPassphrase" # Make sure the wallet isn't encrypted first - address = self.nodes[0].getnewaddress() - privkey = self.nodes[0].dumpprivkey(address) - assert_equal(privkey[:1], "c") - assert_equal(len(privkey), 52) + msg = "test message" + address = self.nodes[0].getnewaddress(address_type='legacy') + sig = self.nodes[0].signmessage(address, msg) + assert self.nodes[0].verifymessage(address, sig, msg) assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrase was called", self.nodes[0].walletpassphrase, 'ff', 1) assert_raises_rpc_error(-15, "Error: running with an unencrypted wallet, but walletpassphrasechange was called.", self.nodes[0].walletpassphrasechange, 'ff', 'ff') @@ -39,33 +38,36 @@ class WalletEncryptionTest(BitcoinTestFramework): self.nodes[0].encryptwallet(passphrase) # Test that the wallet is encrypted - assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].signmessage, address, msg) assert_raises_rpc_error(-15, "Error: running with an encrypted wallet, but encryptwallet was called.", self.nodes[0].encryptwallet, 'ff') assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrase, '', 1) assert_raises_rpc_error(-8, "passphrase can not be empty", self.nodes[0].walletpassphrasechange, '', 'ff') # Check that walletpassphrase works self.nodes[0].walletpassphrase(passphrase, 2) - assert_equal(privkey, self.nodes[0].dumpprivkey(address)) + sig = self.nodes[0].signmessage(address, msg) + assert self.nodes[0].verifymessage(address, sig, msg) # Check that the timeout is right time.sleep(3) - assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].signmessage, address, msg) # Test wrong passphrase assert_raises_rpc_error(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase + "wrong", 10) # Test walletlock self.nodes[0].walletpassphrase(passphrase, 84600) - assert_equal(privkey, self.nodes[0].dumpprivkey(address)) + sig = self.nodes[0].signmessage(address, msg) + assert self.nodes[0].verifymessage(address, sig, msg) self.nodes[0].walletlock() - assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].dumpprivkey, address) + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].signmessage, address, msg) # Test passphrase changes self.nodes[0].walletpassphrasechange(passphrase, passphrase2) assert_raises_rpc_error(-14, "wallet passphrase entered was incorrect", self.nodes[0].walletpassphrase, passphrase, 10) self.nodes[0].walletpassphrase(passphrase2, 10) - assert_equal(privkey, self.nodes[0].dumpprivkey(address)) + sig = self.nodes[0].signmessage(address, msg) + assert self.nodes[0].verifymessage(address, sig, msg) self.nodes[0].walletlock() # Test timeout bounds diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index e4328f2b0e..09f89eb59d 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -27,17 +27,21 @@ class WalletHDTest(BitcoinTestFramework): def run_test(self): # Make sure we use hd, keep masterkeyid - masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - assert_equal(len(masterkeyid), 40) + hd_fingerprint = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['hdmasterfingerprint'] + assert_equal(len(hd_fingerprint), 8) # create an internal key change_addr = self.nodes[1].getrawchangeaddress() change_addrV= self.nodes[1].getaddressinfo(change_addr) - assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key + if self.options.descriptors: + assert_equal(change_addrV["hdkeypath"], "m/84'/1'/0'/1/0") + else: + assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key # Import a non-HD private key in the HD wallet - non_hd_add = self.nodes[0].getnewaddress() - self.nodes[1].importprivkey(self.nodes[0].dumpprivkey(non_hd_add)) + non_hd_add = 'bcrt1qmevj8zfx0wdvp05cqwkmr6mxkfx60yezwjksmt' + non_hd_key = 'cS9umN9w6cDMuRVYdbkfE4c7YUFLJRoXMfhQ569uY4odiQbVN8Rt' + self.nodes[1].importprivkey(non_hd_key) # This should be enough to keep the master key and the non-HD key self.nodes[1].backupwallet(os.path.join(self.nodes[1].datadir, "hd.bak")) @@ -48,11 +52,14 @@ class WalletHDTest(BitcoinTestFramework): self.nodes[0].generate(101) hd_add = None NUM_HD_ADDS = 10 - for i in range(NUM_HD_ADDS): + for i in range(1, NUM_HD_ADDS + 1): hd_add = self.nodes[1].getnewaddress() hd_info = self.nodes[1].getaddressinfo(hd_add) - assert_equal(hd_info["hdkeypath"], "m/0'/0'/"+str(i)+"'") - assert_equal(hd_info["hdseedid"], masterkeyid) + if self.options.descriptors: + assert_equal(hd_info["hdkeypath"], "m/84'/1'/0'/0/" + str(i)) + else: + assert_equal(hd_info["hdkeypath"], "m/0'/0'/"+str(i)+"'") + assert_equal(hd_info["hdmasterfingerprint"], hd_fingerprint) self.nodes[0].sendtoaddress(hd_add, 1) self.nodes[0].generate(1) self.nodes[0].sendtoaddress(non_hd_add, 1) @@ -61,7 +68,10 @@ class WalletHDTest(BitcoinTestFramework): # create an internal key (again) change_addr = self.nodes[1].getrawchangeaddress() change_addrV= self.nodes[1].getaddressinfo(change_addr) - assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key + if self.options.descriptors: + assert_equal(change_addrV["hdkeypath"], "m/84'/1'/0'/1/1") + else: + assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key self.sync_all() assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1) @@ -72,16 +82,19 @@ class WalletHDTest(BitcoinTestFramework): # otherwise node1 would auto-recover all funds in flag the keypool keys as used shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) - shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat")) + shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, 'wallets', "wallet.dat")) self.start_node(1) # Assert that derivation is deterministic hd_add_2 = None - for i in range(NUM_HD_ADDS): + for i in range(1, NUM_HD_ADDS + 1): hd_add_2 = self.nodes[1].getnewaddress() hd_info_2 = self.nodes[1].getaddressinfo(hd_add_2) - assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(i)+"'") - assert_equal(hd_info_2["hdseedid"], masterkeyid) + if self.options.descriptors: + assert_equal(hd_info_2["hdkeypath"], "m/84'/1'/0'/0/" + str(i)) + else: + assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(i)+"'") + assert_equal(hd_info_2["hdmasterfingerprint"], hd_fingerprint) assert_equal(hd_add, hd_add_2) connect_nodes(self.nodes[0], 1) self.sync_all() @@ -117,41 +130,45 @@ class WalletHDTest(BitcoinTestFramework): if out['value'] != 1: keypath = self.nodes[1].getaddressinfo(out['scriptPubKey']['addresses'][0])['hdkeypath'] - assert_equal(keypath[0:7], "m/0'/1'") - - # Generate a new HD seed on node 1 and make sure it is set - orig_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - self.nodes[1].sethdseed() - new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - assert orig_masterkeyid != new_masterkeyid - addr = self.nodes[1].getnewaddress() - assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is the first from the keypool - self.nodes[1].keypoolrefill(1) # Fill keypool with 1 key - - # Set a new HD seed on node 1 without flushing the keypool - new_seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress()) - orig_masterkeyid = new_masterkeyid - self.nodes[1].sethdseed(False, new_seed) - new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - assert orig_masterkeyid != new_masterkeyid - addr = self.nodes[1].getnewaddress() - assert_equal(orig_masterkeyid, self.nodes[1].getaddressinfo(addr)['hdseedid']) - assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/1\'') # Make sure the new address continues previous keypool - - # Check that the next address is from the new seed - self.nodes[1].keypoolrefill(1) - next_addr = self.nodes[1].getnewaddress() - assert_equal(new_masterkeyid, self.nodes[1].getaddressinfo(next_addr)['hdseedid']) - assert_equal(self.nodes[1].getaddressinfo(next_addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is not from previous keypool - assert next_addr != addr - - # Sethdseed parameter validity - assert_raises_rpc_error(-1, 'sethdseed', self.nodes[0].sethdseed, False, new_seed, 0) - assert_raises_rpc_error(-5, "Invalid private key", self.nodes[1].sethdseed, False, "not_wif") - assert_raises_rpc_error(-1, "JSON value is not a boolean as expected", self.nodes[1].sethdseed, "Not_bool") - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[1].sethdseed, False, True) - assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, new_seed) - assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress())) + if self.options.descriptors: + assert_equal(keypath[0:14], "m/84'/1'/0'/1/") + else: + assert_equal(keypath[0:7], "m/0'/1'") + + if not self.options.descriptors: + # Generate a new HD seed on node 1 and make sure it is set + orig_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] + self.nodes[1].sethdseed() + new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] + assert orig_masterkeyid != new_masterkeyid + addr = self.nodes[1].getnewaddress() + assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is the first from the keypool + self.nodes[1].keypoolrefill(1) # Fill keypool with 1 key + + # Set a new HD seed on node 1 without flushing the keypool + new_seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress()) + orig_masterkeyid = new_masterkeyid + self.nodes[1].sethdseed(False, new_seed) + new_masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] + assert orig_masterkeyid != new_masterkeyid + addr = self.nodes[1].getnewaddress() + assert_equal(orig_masterkeyid, self.nodes[1].getaddressinfo(addr)['hdseedid']) + assert_equal(self.nodes[1].getaddressinfo(addr)['hdkeypath'], 'm/0\'/0\'/1\'') # Make sure the new address continues previous keypool + + # Check that the next address is from the new seed + self.nodes[1].keypoolrefill(1) + next_addr = self.nodes[1].getnewaddress() + assert_equal(new_masterkeyid, self.nodes[1].getaddressinfo(next_addr)['hdseedid']) + assert_equal(self.nodes[1].getaddressinfo(next_addr)['hdkeypath'], 'm/0\'/0\'/0\'') # Make sure the new address is not from previous keypool + assert next_addr != addr + + # Sethdseed parameter validity + assert_raises_rpc_error(-1, 'sethdseed', self.nodes[0].sethdseed, False, new_seed, 0) + assert_raises_rpc_error(-5, "Invalid private key", self.nodes[1].sethdseed, False, "not_wif") + assert_raises_rpc_error(-1, "JSON value is not a boolean as expected", self.nodes[1].sethdseed, "Not_bool") + assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[1].sethdseed, False, True) + assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, new_seed) + assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress())) if __name__ == '__main__': WalletHDTest().main () diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py new file mode 100755 index 0000000000..fc5d653a91 --- /dev/null +++ b/test/functional/wallet_importdescriptors.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the importdescriptors RPC. + +Test importdescriptors by generating keys on node0, importing the corresponding +descriptors on node1 and then testing the address info for the different address +variants. + +- `get_generate_key()` is called to generate keys and return the privkeys, + pubkeys and all variants of scriptPubKey and address. +- `test_importdesc()` is called to send an importdescriptors call to node1, test + success, and (if unsuccessful) test the error code and error message returned. +- `test_address()` is called to call getaddressinfo for an address on node1 + and test the values returned.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import descsum_create +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + find_vout_for_address, +) +from test_framework.wallet_util import ( + get_generate_key, + test_address, +) + +class ImportDescriptorsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.extra_args = [["-addresstype=legacy"], + ["-addresstype=bech32", "-keypool=5"] + ] + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_importdesc(self, req, success, error_code=None, error_message=None, warnings=None, wallet=None): + """Run importdescriptors and assert success""" + if warnings is None: + warnings = [] + wrpc = self.nodes[1].get_wallet_rpc('w1') + if wallet is not None: + wrpc = wallet + + result = wrpc.importdescriptors([req]) + observed_warnings = [] + if 'warnings' in result[0]: + observed_warnings = result[0]['warnings'] + assert_equal("\n".join(sorted(warnings)), "\n".join(sorted(observed_warnings))) + assert_equal(result[0]['success'], success) + if error_code is not None: + assert_equal(result[0]['error']['code'], error_code) + assert_equal(result[0]['error']['message'], error_message) + + def run_test(self): + self.log.info('Setting up wallets') + self.nodes[0].createwallet(wallet_name='w0', disable_private_keys=False) + w0 = self.nodes[0].get_wallet_rpc('w0') + + self.nodes[1].createwallet(wallet_name='w1', disable_private_keys=True, blank=True, descriptors=True) + w1 = self.nodes[1].get_wallet_rpc('w1') + assert_equal(w1.getwalletinfo()['keypoolsize'], 0) + + self.nodes[1].createwallet(wallet_name="wpriv", disable_private_keys=False, blank=True, descriptors=True) + wpriv = self.nodes[1].get_wallet_rpc("wpriv") + assert_equal(wpriv.getwalletinfo()['keypoolsize'], 0) + + self.log.info('Mining coins') + w0.generatetoaddress(101, w0.getnewaddress()) + + # RPC importdescriptors ----------------------------------------------- + + # # Test import fails if no descriptor present + key = get_generate_key() + self.log.info("Import should fail if a descriptor is not provided") + self.test_importdesc({"timestamp": "now"}, + success=False, + error_code=-8, + error_message='Descriptor not found.') + + # # Test importing of a P2PKH descriptor + key = get_generate_key() + self.log.info("Should import a p2pkh descriptor") + self.test_importdesc({"desc": descsum_create("pkh(" + key.pubkey + ")"), + "timestamp": "now", + "label": "Descriptor import test"}, + success=True) + test_address(w1, + key.p2pkh_addr, + solvable=True, + ismine=True, + labels=["Descriptor import test"]) + assert_equal(w1.getwalletinfo()['keypoolsize'], 0) + + self.log.info("Internal addresses cannot have labels") + self.test_importdesc({"desc": descsum_create("pkh(" + key.pubkey + ")"), + "timestamp": "now", + "internal": True, + "label": "Descriptor import test"}, + success=False, + error_code=-8, + error_message="Internal addresses should not have a label") + + # # Test importing of a P2SH-P2WPKH descriptor + key = get_generate_key() + self.log.info("Should not import a p2sh-p2wpkh descriptor without checksum") + self.test_importdesc({"desc": "sh(wpkh(" + key.pubkey + "))", + "timestamp": "now" + }, + success=False, + error_code=-5, + error_message="Missing checksum") + + self.log.info("Should not import a p2sh-p2wpkh descriptor that has range specified") + self.test_importdesc({"desc": descsum_create("sh(wpkh(" + key.pubkey + "))"), + "timestamp": "now", + "range": 1, + }, + success=False, + error_code=-8, + error_message="Range should not be specified for an un-ranged descriptor") + + self.log.info("Should not import a p2sh-p2wpkh descriptor and have it set to active") + self.test_importdesc({"desc": descsum_create("sh(wpkh(" + key.pubkey + "))"), + "timestamp": "now", + "active": True, + }, + success=False, + error_code=-8, + error_message="Active descriptors must be ranged") + + self.log.info("Should import a (non-active) p2sh-p2wpkh descriptor") + self.test_importdesc({"desc": descsum_create("sh(wpkh(" + key.pubkey + "))"), + "timestamp": "now", + "active": False, + }, + success=True) + assert_equal(w1.getwalletinfo()['keypoolsize'], 0) + + test_address(w1, + key.p2sh_p2wpkh_addr, + ismine=True, + solvable=True) + + # # Test importing of a multisig descriptor + key1 = get_generate_key() + key2 = get_generate_key() + self.log.info("Should import a 1-of-2 bare multisig from descriptor") + self.test_importdesc({"desc": descsum_create("multi(1," + key1.pubkey + "," + key2.pubkey + ")"), + "timestamp": "now"}, + success=True) + self.log.info("Should not treat individual keys from the imported bare multisig as watchonly") + test_address(w1, + key1.p2pkh_addr, + ismine=False) + + # # Test ranged descriptors + xpriv = "tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg" + xpub = "tpubD6NzVbkrYhZ4YNXVQbNhMK1WqguFsUXceaVJKbmno2aZ3B6QfbMeraaYvnBSGpV3vxLyTTK9DYT1yoEck4XUScMzXoQ2U2oSmE2JyMedq3H" + addresses = ["2N7yv4p8G8yEaPddJxY41kPihnWvs39qCMf", "2MsHxyb2JS3pAySeNUsJ7mNnurtpeenDzLA"] # hdkeypath=m/0'/0'/0' and 1' + addresses += ["bcrt1qrd3n235cj2czsfmsuvqqpr3lu6lg0ju7scl8gn", "bcrt1qfqeppuvj0ww98r6qghmdkj70tv8qpchehegrg8"] # wpkh subscripts corresponding to the above addresses + desc = "sh(wpkh(" + xpub + "/0/0/*" + "))" + + self.log.info("Ranged descriptors cannot have labels") + self.test_importdesc({"desc":descsum_create(desc), + "timestamp": "now", + "range": [0, 100], + "label": "test"}, + success=False, + error_code=-8, + error_message='Ranged descriptors should not have a label') + + self.log.info("Private keys required for private keys enabled wallet") + self.test_importdesc({"desc":descsum_create(desc), + "timestamp": "now", + "range": [0, 100]}, + success=False, + error_code=-4, + error_message='Cannot import descriptor without private keys to a wallet with private keys enabled', + wallet=wpriv) + + self.log.info("Ranged descriptor import should warn without a specified range") + self.test_importdesc({"desc": descsum_create(desc), + "timestamp": "now"}, + success=True, + warnings=['Range not given, using default keypool range']) + assert_equal(w1.getwalletinfo()['keypoolsize'], 0) + + # # Test importing of a ranged descriptor with xpriv + self.log.info("Should not import a ranged descriptor that includes xpriv into a watch-only wallet") + desc = "sh(wpkh(" + xpriv + "/0'/0'/*'" + "))" + self.test_importdesc({"desc": descsum_create(desc), + "timestamp": "now", + "range": 1}, + success=False, + error_code=-4, + error_message='Cannot import private keys to a wallet with private keys disabled') + for address in addresses: + test_address(w1, + address, + ismine=False, + solvable=False) + + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": -1}, + success=False, error_code=-8, error_message='End of range is too high') + + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": [-1, 10]}, + success=False, error_code=-8, error_message='Range should be greater or equal than 0') + + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": [(2 << 31 + 1) - 1000000, (2 << 31 + 1)]}, + success=False, error_code=-8, error_message='End of range is too high') + + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": [2, 1]}, + success=False, error_code=-8, error_message='Range specified as [begin,end] must not have begin after end') + + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": [0, 1000001]}, + success=False, error_code=-8, error_message='Range is too large') + + # Make sure ranged imports import keys in order + w1 = self.nodes[1].get_wallet_rpc('w1') + self.log.info('Key ranges should be imported in order') + xpub = "tpubDAXcJ7s7ZwicqjprRaEWdPoHKrCS215qxGYxpusRLLmJuT69ZSicuGdSfyvyKpvUNYBW1s2U3NSrT6vrCYB9e6nZUEvrqnwXPF8ArTCRXMY" + addresses = [ + 'bcrt1qtmp74ayg7p24uslctssvjm06q5phz4yrxucgnv', # m/0'/0'/0 + 'bcrt1q8vprchan07gzagd5e6v9wd7azyucksq2xc76k8', # m/0'/0'/1 + 'bcrt1qtuqdtha7zmqgcrr26n2rqxztv5y8rafjp9lulu', # m/0'/0'/2 + 'bcrt1qau64272ymawq26t90md6an0ps99qkrse58m640', # m/0'/0'/3 + 'bcrt1qsg97266hrh6cpmutqen8s4s962aryy77jp0fg0', # m/0'/0'/4 + ] + + self.test_importdesc({'desc': descsum_create('wpkh([80002067/0h/0h]' + xpub + '/*)'), + 'active': True, + 'range' : [0, 2], + 'timestamp': 'now' + }, + success=True) + self.test_importdesc({'desc': descsum_create('sh(wpkh([abcdef12/0h/0h]' + xpub + '/*))'), + 'active': True, + 'range' : [0, 2], + 'timestamp': 'now' + }, + success=True) + self.test_importdesc({'desc': descsum_create('pkh([12345678/0h/0h]' + xpub + '/*)'), + 'active': True, + 'range' : [0, 2], + 'timestamp': 'now' + }, + success=True) + + assert_equal(w1.getwalletinfo()['keypoolsize'], 5 * 3) + for i, expected_addr in enumerate(addresses): + received_addr = w1.getnewaddress('', 'bech32') + assert_raises_rpc_error(-4, 'This wallet has no available keys', w1.getrawchangeaddress, 'bech32') + assert_equal(received_addr, expected_addr) + bech32_addr_info = w1.getaddressinfo(received_addr) + assert_equal(bech32_addr_info['desc'][:23], 'wpkh([80002067/0\'/0\'/{}]'.format(i)) + + shwpkh_addr = w1.getnewaddress('', 'p2sh-segwit') + shwpkh_addr_info = w1.getaddressinfo(shwpkh_addr) + assert_equal(shwpkh_addr_info['desc'][:26], 'sh(wpkh([abcdef12/0\'/0\'/{}]'.format(i)) + + pkh_addr = w1.getnewaddress('', 'legacy') + pkh_addr_info = w1.getaddressinfo(pkh_addr) + assert_equal(pkh_addr_info['desc'][:22], 'pkh([12345678/0\'/0\'/{}]'.format(i)) + + assert_equal(w1.getwalletinfo()['keypoolsize'], 4 * 3) # After retrieving a key, we don't refill the keypool again, so it's one less for each address type + w1.keypoolrefill() + assert_equal(w1.getwalletinfo()['keypoolsize'], 5 * 3) + + # Check active=False default + self.log.info('Check imported descriptors are not active by default') + self.test_importdesc({'desc': descsum_create('pkh([12345678/0h/0h]' + xpub + '/*)'), + 'range' : [0, 2], + 'timestamp': 'now', + 'internal': True + }, + success=True) + assert_raises_rpc_error(-4, 'This wallet has no available keys', w1.getrawchangeaddress, 'legacy') + + # # Test importing a descriptor containing a WIF private key + wif_priv = "cTe1f5rdT8A8DFgVWTjyPwACsDPJM9ff4QngFxUixCSvvbg1x6sh" + address = "2MuhcG52uHPknxDgmGPsV18jSHFBnnRgjPg" + desc = "sh(wpkh(" + wif_priv + "))" + self.log.info("Should import a descriptor with a WIF private key as spendable") + self.test_importdesc({"desc": descsum_create(desc), + "timestamp": "now"}, + success=True, + wallet=wpriv) + test_address(wpriv, + address, + solvable=True, + ismine=True) + txid = w0.sendtoaddress(address, 49.99995540) + w0.generatetoaddress(6, w0.getnewaddress()) + self.sync_blocks() + tx = wpriv.createrawtransaction([{"txid": txid, "vout": 0}], {w0.getnewaddress(): 49.999}) + signed_tx = wpriv.signrawtransactionwithwallet(tx) + w1.sendrawtransaction(signed_tx['hex']) + + # Make sure that we can use import and use multisig as addresses + self.log.info('Test that multisigs can be imported, signed for, and getnewaddress\'d') + self.nodes[1].createwallet(wallet_name="wmulti_priv", disable_private_keys=False, blank=True, descriptors=True) + wmulti_priv = self.nodes[1].get_wallet_rpc("wmulti_priv") + assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 0) + + self.test_importdesc({"desc":"wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/0h/0h/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/0h/0h/*,tprv8ZgxMBicQKsPeonDt8Ka2mrQmHa61hQ5FQCsvWBTpSNzBFgM58cV2EuXNAHF14VawVpznnme3SuTbA62sGriwWyKifJmXntfNeK7zeqMCj1/84h/0h/0h/*))#m2sr93jn", + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now"}, + success=True, + wallet=wmulti_priv) + self.test_importdesc({"desc":"wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/1h/0h/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/1h/0h/*,tprv8ZgxMBicQKsPeonDt8Ka2mrQmHa61hQ5FQCsvWBTpSNzBFgM58cV2EuXNAHF14VawVpznnme3SuTbA62sGriwWyKifJmXntfNeK7zeqMCj1/84h/1h/0h/*))#q3sztvx5", + "active": True, + "internal" : True, + "range": 1000, + "next_index": 0, + "timestamp": "now"}, + success=True, + wallet=wmulti_priv) + + assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 1001) # Range end (1000) is inclusive, so 1001 addresses generated + addr = wmulti_priv.getnewaddress('', 'bech32') + assert_equal(addr, 'bcrt1qdt0qy5p7dzhxzmegnn4ulzhard33s2809arjqgjndx87rv5vd0fq2czhy8') # Derived at m/84'/0'/0'/0 + change_addr = wmulti_priv.getrawchangeaddress('bech32') + assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') + assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 1000) + txid = w0.sendtoaddress(addr, 10) + self.nodes[0].generate(6) + self.sync_all() + send_txid = wmulti_priv.sendtoaddress(w0.getnewaddress(), 8) + decoded = wmulti_priv.decoderawtransaction(wmulti_priv.gettransaction(send_txid)['hex']) + assert_equal(len(decoded['vin'][0]['txinwitness']), 4) + self.nodes[0].generate(6) + self.sync_all() + + self.nodes[1].createwallet(wallet_name="wmulti_pub", disable_private_keys=True, blank=True, descriptors=True) + wmulti_pub = self.nodes[1].get_wallet_rpc("wmulti_pub") + assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 0) + + self.test_importdesc({"desc":"wsh(multi(2,[7b2d0242/84h/0h/0h]tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*,[59b09cd6/84h/0h/0h]tpubDDBF2BTR6s8drwrfDei8WxtckGuSm1cyoKxYY1QaKSBFbHBYQArWhHPA6eJrzZej6nfHGLSURYSLHr7GuYch8aY5n61tGqgn8b4cXrMuoPH/*,[e81a0532/84h/0h/0h]tpubDCsWoW1kuQB9kG5MXewHqkbjPtqPueRnXju7uM2NK7y3JYb2ajAZ9EiuZXNNuE4661RAfriBWhL8UsnAPpk8zrKKnZw1Ug7X4oHgMdZiU4E/*))#tsry0s5e", + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now"}, + success=True, + wallet=wmulti_pub) + self.test_importdesc({"desc":"wsh(multi(2,[7b2d0242/84h/1h/0h]tpubDCXqdwWZcszwqYJSnZp8eARkxGJfHAk23KDxbztV4BbschfaTfYLTcSkSJ3TN64dRqwa1rnFUScsYormKkGqNbbPwkorQimVevXjxzUV9Gf/*,[59b09cd6/84h/1h/0h]tpubDCYfZY2ceyHzYzMMVPt9MNeiqtQ2T7Uyp9QSFwYXh8Vi9iJFYXcuphJaGXfF3jUQJi5Y3GMNXvM11gaL4txzZgNGK22BFAwMXynnzv4z2Jh/*,[e81a0532/84h/1h/0h]tpubDC6UGqnsQStngYuGD4MKsMy7eD1Yg9NTJfPdvjdG2JE5oZ7EsSL3WHg4Gsw2pR5K39ZwJ46M1wZayhedVdQtMGaUhq5S23PH6fnENK3V1sb/*))#c08a2rzv", + "active": True, + "internal" : True, + "range": 1000, + "next_index": 0, + "timestamp": "now"}, + success=True, + wallet=wmulti_pub) + + assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 1000) # The first one was already consumed by previous import and is detected as used + addr = wmulti_pub.getnewaddress('', 'bech32') + assert_equal(addr, 'bcrt1qp8s25ckjl7gr6x2q3dx3tn2pytwp05upkjztk6ey857tt50r5aeqn6mvr9') # Derived at m/84'/0'/0'/1 + change_addr = wmulti_pub.getrawchangeaddress('bech32') + assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') + assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 999) + txid = w0.sendtoaddress(addr, 10) + vout = find_vout_for_address(self.nodes[0], txid, addr) + self.nodes[0].generate(6) + self.sync_all() + assert_equal(wmulti_pub.getbalance(), wmulti_priv.getbalance()) + + self.log.info("Multisig with distributed keys") + self.nodes[1].createwallet(wallet_name="wmulti_priv1", descriptors=True) + wmulti_priv1 = self.nodes[1].get_wallet_rpc("wmulti_priv1") + res = wmulti_priv1.importdescriptors([ + { + "desc": descsum_create("wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/0h/0h/*,[59b09cd6/84h/0h/0h]tpubDDBF2BTR6s8drwrfDei8WxtckGuSm1cyoKxYY1QaKSBFbHBYQArWhHPA6eJrzZej6nfHGLSURYSLHr7GuYch8aY5n61tGqgn8b4cXrMuoPH/*,[e81a0532/84h/0h/0h]tpubDCsWoW1kuQB9kG5MXewHqkbjPtqPueRnXju7uM2NK7y3JYb2ajAZ9EiuZXNNuE4661RAfriBWhL8UsnAPpk8zrKKnZw1Ug7X4oHgMdZiU4E/*))"), + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }, + { + "desc": descsum_create("wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/1h/0h/*,[59b09cd6/84h/1h/0h]tpubDCYfZY2ceyHzYzMMVPt9MNeiqtQ2T7Uyp9QSFwYXh8Vi9iJFYXcuphJaGXfF3jUQJi5Y3GMNXvM11gaL4txzZgNGK22BFAwMXynnzv4z2Jh/*,[e81a0532/84h/1h/0h]tpubDC6UGqnsQStngYuGD4MKsMy7eD1Yg9NTJfPdvjdG2JE5oZ7EsSL3WHg4Gsw2pR5K39ZwJ46M1wZayhedVdQtMGaUhq5S23PH6fnENK3V1sb/*))"), + "active": True, + "internal" : True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }]) + assert_equal(res[0]['success'], True) + assert_equal(res[0]['warnings'][0], 'Not all private keys provided. Some wallet functionality may return unexpected errors') + assert_equal(res[1]['success'], True) + assert_equal(res[1]['warnings'][0], 'Not all private keys provided. Some wallet functionality may return unexpected errors') + + self.nodes[1].createwallet(wallet_name='wmulti_priv2', blank=True, descriptors=True) + wmulti_priv2 = self.nodes[1].get_wallet_rpc('wmulti_priv2') + res = wmulti_priv2.importdescriptors([ + { + "desc": descsum_create("wsh(multi(2,[7b2d0242/84h/0h/0h]tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/0h/0h/*,[e81a0532/84h/0h/0h]tpubDCsWoW1kuQB9kG5MXewHqkbjPtqPueRnXju7uM2NK7y3JYb2ajAZ9EiuZXNNuE4661RAfriBWhL8UsnAPpk8zrKKnZw1Ug7X4oHgMdZiU4E/*))"), + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }, + { + "desc": descsum_create("wsh(multi(2,[7b2d0242/84h/1h/0h]tpubDCXqdwWZcszwqYJSnZp8eARkxGJfHAk23KDxbztV4BbschfaTfYLTcSkSJ3TN64dRqwa1rnFUScsYormKkGqNbbPwkorQimVevXjxzUV9Gf/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/1h/0h/*,[e81a0532/84h/1h/0h]tpubDC6UGqnsQStngYuGD4MKsMy7eD1Yg9NTJfPdvjdG2JE5oZ7EsSL3WHg4Gsw2pR5K39ZwJ46M1wZayhedVdQtMGaUhq5S23PH6fnENK3V1sb/*))"), + "active": True, + "internal" : True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }]) + assert_equal(res[0]['success'], True) + assert_equal(res[0]['warnings'][0], 'Not all private keys provided. Some wallet functionality may return unexpected errors') + assert_equal(res[1]['success'], True) + assert_equal(res[1]['warnings'][0], 'Not all private keys provided. Some wallet functionality may return unexpected errors') + + rawtx = self.nodes[1].createrawtransaction([{'txid': txid, 'vout': vout}], {w0.getnewaddress(): 9.999}) + tx_signed_1 = wmulti_priv1.signrawtransactionwithwallet(rawtx) + assert_equal(tx_signed_1['complete'], False) + tx_signed_2 = wmulti_priv2.signrawtransactionwithwallet(tx_signed_1['hex']) + assert_equal(tx_signed_2['complete'], True) + self.nodes[1].sendrawtransaction(tx_signed_2['hex']) + + self.log.info("Combo descriptors cannot be active") + self.test_importdesc({"desc": descsum_create("combo(tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*)"), + "active": True, + "range": 1, + "timestamp": "now"}, + success=False, + error_code=-4, + error_message="Combo descriptors cannot be set to active") + + self.log.info("Descriptors with no type cannot be active") + self.test_importdesc({"desc": descsum_create("pk(tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*)"), + "active": True, + "range": 1, + "timestamp": "now"}, + success=True, + warnings=["Unknown output type, cannot set descriptor to active."]) + +if __name__ == '__main__': + ImportDescriptorsTest().main() diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index 9e2f00e62f..40a2b3ab6a 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -22,16 +22,63 @@ class KeyPoolTest(BitcoinTestFramework): addr_before_encrypting = nodes[0].getnewaddress() addr_before_encrypting_data = nodes[0].getaddressinfo(addr_before_encrypting) wallet_info_old = nodes[0].getwalletinfo() - assert addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid'] + if not self.options.descriptors: + assert addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid'] # Encrypt wallet and wait to terminate nodes[0].encryptwallet('test') + if self.options.descriptors: + # Import hardened derivation only descriptors + nodes[0].walletpassphrase('test', 10) + nodes[0].importdescriptors([ + { + "desc": "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/*h)#y4dfsj7n", + "timestamp": "now", + "range": [0,0], + "active": True + }, + { + "desc": "pkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1h/*h)#a0nyvl0k", + "timestamp": "now", + "range": [0,0], + "active": True + }, + { + "desc": "sh(wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/2h/*h))#lmeu2axg", + "timestamp": "now", + "range": [0,0], + "active": True + }, + { + "desc": "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/3h/*h)#jkl636gm", + "timestamp": "now", + "range": [0,0], + "active": True, + "internal": True + }, + { + "desc": "pkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/4h/*h)#l3crwaus", + "timestamp": "now", + "range": [0,0], + "active": True, + "internal": True + }, + { + "desc": "sh(wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/5h/*h))#qg8wa75f", + "timestamp": "now", + "range": [0,0], + "active": True, + "internal": True + } + ]) + nodes[0].walletlock() # Keep creating keys addr = nodes[0].getnewaddress() addr_data = nodes[0].getaddressinfo(addr) wallet_info = nodes[0].getwalletinfo() - assert addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid'] - assert addr_data['hdseedid'] == wallet_info['hdseedid'] + assert addr_before_encrypting_data['hdmasterfingerprint'] != addr_data['hdmasterfingerprint'] + if not self.options.descriptors: + assert addr_data['hdseedid'] == wallet_info['hdseedid'] assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) # put six (plus 2) new keys in the keypool (100% external-, +100% internal-keys, 1 in min) @@ -39,8 +86,12 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0].keypoolrefill(6) nodes[0].walletlock() wi = nodes[0].getwalletinfo() - assert_equal(wi['keypoolsize_hd_internal'], 6) - assert_equal(wi['keypoolsize'], 6) + if self.options.descriptors: + assert_equal(wi['keypoolsize_hd_internal'], 18) + assert_equal(wi['keypoolsize'], 18) + else: + assert_equal(wi['keypoolsize_hd_internal'], 6) + assert_equal(wi['keypoolsize'], 6) # drain the internal keys nodes[0].getrawchangeaddress() @@ -80,11 +131,15 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0].walletpassphrase('test', 100) nodes[0].keypoolrefill(100) wi = nodes[0].getwalletinfo() - assert_equal(wi['keypoolsize_hd_internal'], 100) - assert_equal(wi['keypoolsize'], 100) + if self.options.descriptors: + assert_equal(wi['keypoolsize_hd_internal'], 300) + assert_equal(wi['keypoolsize'], 300) + else: + assert_equal(wi['keypoolsize_hd_internal'], 100) + assert_equal(wi['keypoolsize'], 100) # create a blank wallet - nodes[0].createwallet(wallet_name='w2', blank=True) + nodes[0].createwallet(wallet_name='w2', blank=True, disable_private_keys=True) w2 = nodes[0].get_wallet_rpc('w2') # refer to initial wallet as w1 @@ -92,8 +147,11 @@ class KeyPoolTest(BitcoinTestFramework): # import private key and fund it address = addr.pop() - privkey = w1.dumpprivkey(address) - res = w2.importmulti([{'scriptPubKey': {'address': address}, 'keys': [privkey], 'timestamp': 'now'}]) + desc = w1.getaddressinfo(address)['desc'] + if self.options.descriptors: + res = w2.importdescriptors([{'desc': desc, 'timestamp': 'now'}]) + else: + res = w2.importmulti([{'desc': desc, 'timestamp': 'now'}]) assert_equal(res[0]['success'], True) w1.walletpassphrase('test', 100) diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index 829633a050..102ed23fba 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -79,7 +79,15 @@ class KeypoolRestoreTest(BitcoinTestFramework): assert_equal(self.nodes[idx].getbalance(), 15) assert_equal(self.nodes[idx].listtransactions()[0]['category'], "receive") # Check that we have marked all keys up to the used keypool key as used - assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress())['hdkeypath'], "m/0'/0'/110'") + if self.options.descriptors: + if output_type == 'legacy': + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/44'/1'/0'/0/110") + elif output_type == 'p2sh-segwit': + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/49'/1'/0'/0/110") + elif output_type == 'bech32': + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/84'/1'/0'/0/110") + else: + assert_equal(self.nodes[idx].getaddressinfo(self.nodes[idx].getnewaddress(address_type=output_type))['hdkeypath'], "m/0'/0'/110'") if __name__ == '__main__': diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index 337d2e55d9..f8d1720469 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -115,15 +115,16 @@ class WalletLabelsTest(BitcoinTestFramework): assert_raises_rpc_error(-11, "No addresses with label", node.getaddressesbylabel, "") # Check that addmultisigaddress can assign labels. - for label in labels: - addresses = [] - for x in range(10): - addresses.append(node.getnewaddress()) - multisig_address = node.addmultisigaddress(5, addresses, label.name)['address'] - label.add_address(multisig_address) - label.purpose[multisig_address] = "send" - label.verify(node) - node.generate(101) + if not self.options.descriptors: + for label in labels: + addresses = [] + for x in range(10): + addresses.append(node.getnewaddress()) + multisig_address = node.addmultisigaddress(5, addresses, label.name)['address'] + label.add_address(multisig_address) + label.purpose[multisig_address] = "send" + label.verify(node) + node.generate(101) # Check that setlabel can change the label of an address from a # different label. diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index c569416292..580a61f9f3 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -227,10 +227,10 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-18, 'Wallet wallets not found.', self.nodes[0].loadwallet, 'wallets') # Fail to load duplicate wallets - assert_raises_rpc_error(-4, 'Wallet file verification failed: Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0]) + assert_raises_rpc_error(-4, 'Wallet file verification failed. Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0]) # Fail to load duplicate wallets by different ways (directory and filepath) - assert_raises_rpc_error(-4, "Wallet file verification failed: Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') + assert_raises_rpc_error(-4, "Wallet file verification failed. Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') # Fail to load if one wallet is a copy of another assert_raises_rpc_error(-4, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') @@ -240,7 +240,7 @@ class MultiWalletTest(BitcoinTestFramework): # Fail to load if wallet file is a symlink - assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') + assert_raises_rpc_error(-4, "Wallet file verification failed. Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') # Fail to load if a directory is specified that doesn't contain a wallet os.mkdir(wallet_dir('empty_wallet_dir')) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index db5902f820..b384998d56 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -3,29 +3,14 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet resends transactions periodically.""" -from collections import defaultdict import time from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import ToHex -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.mininode import P2PTxInvStore, mininode_lock from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, wait_until - -class P2PStoreTxInvs(P2PInterface): - def __init__(self): - super().__init__() - self.tx_invs_received = defaultdict(int) - - def on_inv(self, message): - # Store how many times invs have been received for each tx. - for i in message.inv: - if i.type == 1: - # save txid - self.tx_invs_received[i.hash] += 1 - - class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -36,7 +21,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] # alias - node.add_p2p_connection(P2PStoreTxInvs()) + node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a new transaction and wait until it's broadcast") txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) @@ -51,7 +36,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock) # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) - node.add_p2p_connection(P2PStoreTxInvs()) + node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a block") # Create and submit a block without the transaction. @@ -69,9 +54,10 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): node.p2ps[1].sync_with_ping() assert_equal(node.p2ps[1].tx_invs_received[txid], 0) - self.log.info("Transaction should be rebroadcast after 30 minutes") - # Use mocktime and give an extra 5 minutes to be sure. - rebroadcast_time = int(time.time()) + 41 * 60 + self.log.info("Bump time & check that transaction is rebroadcast") + # Transaction should be rebroadcast approximately 24 hours in the future, + # but can range from 12-36. So bump 36 hours to be sure. + rebroadcast_time = int(time.time()) + 36 * 60 * 60 node.setmocktime(rebroadcast_time) wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py new file mode 100755 index 0000000000..e7e71bf3f6 --- /dev/null +++ b/test/functional/wallet_upgradewallet.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""upgradewallet RPC functional test + +Test upgradewallet RPC. Download v0.15.2 v0.16.3 node binaries: + +contrib/devtools/previous_release.sh -b v0.15.2 v0.16.3 +""" + +import os +import shutil + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + adjust_bitcoin_conf_for_pre_17, + assert_equal, + assert_greater_than, + assert_is_hex_string, +) + + +class UpgradeWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 3 + self.extra_args = [ + ["-addresstype=bech32"], # current wallet version + ["-usehd=1"], # v0.16.3 wallet + ["-usehd=0"] # v0.15.2 wallet + ] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_previous_releases() + + def setup_network(self): + self.setup_nodes() + + def setup_nodes(self): + self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ + None, + 160300, + 150200, + ]) + # adapt bitcoin.conf, because older bitcoind's don't recognize config sections + adjust_bitcoin_conf_for_pre_17(self.nodes[1].bitcoinconf) + adjust_bitcoin_conf_for_pre_17(self.nodes[2].bitcoinconf) + self.start_nodes() + + def dumb_sync_blocks(self): + """ + Little helper to sync older wallets. + Notice that v0.15.2's regtest is hardforked, so there is + no sync for it. + v0.15.2 is only being used to test for version upgrade + and master hash key presence. + v0.16.3 is being used to test for version upgrade and balances. + Further info: https://github.com/bitcoin/bitcoin/pull/18774#discussion_r416967844 + """ + node_from = self.nodes[0] + v16_3_node = self.nodes[1] + to_height = node_from.getblockcount() + height = self.nodes[1].getblockcount() + for i in range(height, to_height+1): + b = node_from.getblock(blockhash=node_from.getblockhash(i), verbose=0) + v16_3_node.submitblock(b) + assert_equal(v16_3_node.getblockcount(), to_height) + + def run_test(self): + self.nodes[0].generatetoaddress(101, self.nodes[0].getnewaddress()) + self.dumb_sync_blocks() + # # Sanity check the test framework: + res = self.nodes[0].getblockchaininfo() + assert_equal(res['blocks'], 101) + node_master = self.nodes[0] + v16_3_node = self.nodes[1] + v15_2_node = self.nodes[2] + + # Send coins to old wallets for later conversion checks. + v16_3_wallet = v16_3_node.get_wallet_rpc('wallet.dat') + v16_3_address = v16_3_wallet.getnewaddress() + node_master.generatetoaddress(101, v16_3_address) + self.dumb_sync_blocks() + v16_3_balance = v16_3_wallet.getbalance() + + self.log.info("Test upgradewallet RPC...") + # Prepare for copying of the older wallet + node_master_wallet_dir = os.path.join(node_master.datadir, "regtest/wallets") + v16_3_wallet = os.path.join(v16_3_node.datadir, "regtest/wallets/wallet.dat") + v15_2_wallet = os.path.join(v15_2_node.datadir, "regtest/wallet.dat") + self.stop_nodes() + + # Copy the 0.16.3 wallet to the last Bitcoin Core version and open it: + shutil.rmtree(node_master_wallet_dir) + os.mkdir(node_master_wallet_dir) + shutil.copy( + v16_3_wallet, + node_master_wallet_dir + ) + self.restart_node(0, ['-nowallet']) + node_master.loadwallet('') + + wallet = node_master.get_wallet_rpc('') + old_version = wallet.getwalletinfo()["walletversion"] + + # calling upgradewallet without version arguments + # should return nothing if successful + assert_equal(wallet.upgradewallet(), "") + new_version = wallet.getwalletinfo()["walletversion"] + # upgraded wallet version should be greater than older one + assert_greater_than(new_version, old_version) + # wallet should still contain the same balance + assert_equal(wallet.getbalance(), v16_3_balance) + + self.stop_node(0) + # Copy the 0.15.2 wallet to the last Bitcoin Core version and open it: + shutil.rmtree(node_master_wallet_dir) + os.mkdir(node_master_wallet_dir) + shutil.copy( + v15_2_wallet, + node_master_wallet_dir + ) + self.restart_node(0, ['-nowallet']) + node_master.loadwallet('') + + wallet = node_master.get_wallet_rpc('') + # should have no master key hash before conversion + assert_equal('hdseedid' in wallet.getwalletinfo(), False) + # calling upgradewallet with explicit version number + # should return nothing if successful + assert_equal(wallet.upgradewallet(169900), "") + new_version = wallet.getwalletinfo()["walletversion"] + # upgraded wallet should have version 169900 + assert_equal(new_version, 169900) + # after conversion master key hash should be present + assert_is_hex_string(wallet.getwalletinfo()['hdseedid']) + +if __name__ == '__main__': + UpgradeWalletTest().main() diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 5540a0f74f..2bb76ec286 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -46,15 +46,17 @@ if ! command -v yq > /dev/null; then fi EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")" +SHELLCHECK_CMD="shellcheck --external-sources --check-sourced $EXCLUDE_GITIAN" for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml') do - echo - echo "$descriptor" + script=$(basename "$descriptor") # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script. - SCRIPT=$'#!/bin/bash\n'$(yq -r .script "$descriptor") - if ! echo "$SCRIPT" | shellcheck "$EXCLUDE_GITIAN" -; then + echo "#!/bin/bash" > $script + yq -r .script "$descriptor" >> $script + if ! $SHELLCHECK_CMD $script; then EXIT_CODE=1 fi + rm $script done exit $EXIT_CODE |