diff options
359 files changed, 6624 insertions, 3779 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index b839bfa5fb..fde5551531 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,33 +1,3 @@ -task: - name: "FreeBsd 12.1 amd64 [GOAL: install] [no depends, only system libs]" - freebsd_instance: - image_family: freebsd-12-1 # https://cirrus-ci.org/guide/FreeBSD/ - cpu: 8 - memory: 8G - timeout_in: 60m - env: - MAKEJOBS: "-j9" - CONFIGURE_OPTS: "--disable-dependency-tracking" - GOAL: "install" - TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache - CCACHE_SIZE: "200M" - CCACHE_COMPRESS: 1 - CCACHE_DIR: "/tmp/ccache_dir" - ccache_cache: - folder: "/tmp/ccache_dir" - install_script: - - pkg install -y autoconf automake boost-libs git gmake libevent libtool pkgconf python3 ccache - - ./contrib/install_db4.sh $(pwd) - - ccache --max-size=${CCACHE_SIZE} - configure_script: - - ./autogen.sh - - ./configure ${CONFIGURE_OPTS} BDB_LIBS="-L$(pwd)/db4/lib -ldb_cxx-4.8" BDB_CFLAGS="-I$(pwd)/db4/include" || ( cat config.log && false) - make_script: - - gmake ${MAKEJOBS} ${GOAL} || ( echo "Build failure. Verbose build follows." && gmake ${GOAL} V=1 ; false ) - check_script: - - gmake check ${MAKEJOBS} VERBOSE=1 - functional_test_script: - - ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast #task: # name: "Windows" # windows_container: diff --git a/.fuzzbuzz.yml b/.fuzzbuzz.yml new file mode 100644 index 0000000000..d44ac27eb9 --- /dev/null +++ b/.fuzzbuzz.yml @@ -0,0 +1,16 @@ +base: ubuntu:16.04 +language: c++ +engine: libFuzzer +environment: + - CXXFLAGS=-fcoverage-mapping -fno-omit-frame-pointer -fprofile-instr-generate -gline-tables-only -O1 +setup: + - sudo apt-get update + - sudo apt-get install -y autoconf bsdmainutils clang git libboost-all-dev libboost-program-options-dev libc++1 libc++abi1 libc++abi-dev libc++-dev libclang1 libclang-dev libdb5.3++ libevent-dev libllvm-ocaml-dev libomp5 libomp-dev libprotobuf-dev libqt5core5a libqt5dbus5 libqt5gui5 libssl-dev libtool llvm llvm-dev llvm-runtime pkg-config protobuf-compiler qttools5-dev qttools5-dev-tools software-properties-common + - ./autogen.sh + - CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined + - make + - git clone https://github.com/bitcoin-core/qa-assets +auto_targets: + find_targets_command: find src/test/fuzz/ -executable -type f ! -name "*.cpp" ! -name "*.h" + base_corpus_dir: qa-assets/fuzz_seed_corpus/ + memory_limit: none diff --git a/.gitignore b/.gitignore index 1c487f43a7..5b9ec1f355 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ src/bitcoin src/bitcoind src/bitcoin-cli +src/bitcoin-gui +src/bitcoin-node src/bitcoin-tx src/bitcoin-wallet src/test/fuzz diff --git a/.travis.yml b/.travis.yml index ea589b4059..1598813ed2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,6 @@ stages: env: global: - CI_RETRY_EXE="travis_retry" - - CI_WAIT="while sleep 500; do echo .; done" - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. See https://docs.travis-ci.com/user/customizing-the-build#build-timeouts . Please manually re-run this job by using the travis restart button. The next run should not time out because the build cache has been saved." before_install: - set -o errexit; source ./ci/test/00_setup_env.sh @@ -88,26 +87,6 @@ jobs: FILE_ENV="./ci/test/00_setup_env_arm.sh" QEMU_USER_CMD="" - - stage: test - name: 's390x native BE [GOAL: install] [bionic] [no depends, no GUI]' - arch: s390x - dist: bionic - addons: - apt: - packages: - - bsdmainutils - - libboost-filesystem-dev - - libboost-system-dev - - libboost-test-dev - - libboost-thread-dev - - libdb++-dev - - libdb-dev - - libevent-dev - env: >- - DANGER_RUN_CI_ON_HOST=true - CI_USE_APT_INSTALL=no - FILE_ENV="./ci/test/00_setup_env_s390x_host.sh" - # s390 build was disabled temporarily because of disk space issues on the Travis VM # # - stage: test @@ -128,15 +107,16 @@ jobs: FILE_ENV="./ci/test/00_setup_env_i686_centos.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [bionic] [previous releases, uses qt5 dev package and some depends packages] [unsigned char]' + name: 'x86_64 Linux [GOAL: install] [bionic] [C++17, previous releases, uses qt5 dev package and some depends packages] [unsigned char]' env: >- FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [xenial] [no depends, only system libs, sanitizers: thread (TSan), no wallet]' + name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: thread (TSan), no wallet]' + # Not enough memory on travis machines, so feature_block is excluded for now env: >- + TEST_RUNNER_EXTRA="--exclude feature_block" FILE_ENV="./ci/test/00_setup_env_native_tsan.sh" - TEST_RUNNER_EXTRA="--exclude feature_block" # Not enough memory on travis machines - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' @@ -144,19 +124,15 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_asan.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, valgrind]' - env: >- - FILE_ENV="./ci/test/00_setup_env_native_valgrind.sh" - - - stage: test name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, sanitizers: fuzzer,address,undefined]' env: >- FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, fuzzers under valgrind]' + name: 'x86_64 Linux [GOAL: install] [bionic] [multiprocess]' + if: type != pull_request OR commit_message =~ /depends:|multiprocess:/ # Skip on non-depends, non-multiprocess PRs env: >- - FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" + FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]' diff --git a/Makefile.am b/Makefile.am index c35f5080aa..45dab3930d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -41,7 +41,7 @@ OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus OSX_FANCY_PLIST=$(top_srcdir)/contrib/macdeploy/fancy.plist OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed -OSX_QT_TRANSLATIONS = da,de,es,hu,ru,uk,zh_CN,zh_TW +OSX_QT_TRANSLATIONS = ar,bg,ca,cs,da,de,es,fa,fi,fr,gd,gl,he,hu,it,ja,ko,lt,lv,pl,pt,ru,sk,sl,sv,uk,zh_CN,zh_TW DIST_CONTRIB = \ $(top_srcdir)/contrib/linearize/linearize-data.py \ @@ -80,7 +80,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release - @test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \ + @test -f $(MAKENSIS) && echo 'OutFile "$@"' | cat $(top_builddir)/share/setup.nsi - | $(MAKENSIS) -V2 - || \ echo error: could not build $@ @echo built $@ diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 index f147cee3b1..43087b2e68 100644 --- a/build-aux/m4/ax_cxx_compile_stdcxx.m4 +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html # =========================================================================== # # SYNOPSIS @@ -33,21 +33,23 @@ # Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com> # Copyright (c) 2015 Paul Norman <penorman@mac.com> # Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> +# Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com> +# Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com> # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 4 +#serial 11 dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro dnl (serial version number 13). AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl - m4_if([$1], [11], [], - [$1], [14], [], - [$1], [17], [m4_fatal([support for C++17 not yet implemented in AX_CXX_COMPILE_STDCXX])], + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl m4_if([$2], [], [], [$2], [ext], [], @@ -57,26 +59,13 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], [$3], [optional], [ax_cxx_compile_cxx$1_required=false], [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) - m4_if([$4], [], [ax_cxx_compile_cxx$1_try_default=true], - [$4], [default], [ax_cxx_compile_cxx$1_try_default=true], - [$4], [nodefault], [ax_cxx_compile_cxx$1_try_default=false], - [m4_fatal([invalid fourth argument `$4' to AX_CXX_COMPILE_STDCXX])]) AC_LANG_PUSH([C++])dnl ac_success=no - m4_if([$4], [nodefault], [], [dnl - AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, - ax_cv_cxx_compile_cxx$1, - [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], - [ax_cv_cxx_compile_cxx$1=yes], - [ax_cv_cxx_compile_cxx$1=no])]) - if test x$ax_cv_cxx_compile_cxx$1 = xyes; then - ac_success=yes - fi]) - m4_if([$2], [noext], [], [dnl if test x$ac_success = xno; then - for switch in -std=gnu++$1 -std=gnu++0x; do + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, $cachevar, @@ -102,22 +91,27 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl dnl HP's aCC needs +std=c++11 according to: dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf dnl Cray's crayCC needs "-h std=c++11" - for switch in -std=c++$1 -std=c++0x +std=c++$1 "-h std=c++$1"; do - cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) - AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, - $cachevar, - [ac_save_CXX="$CXX" - CXX="$CXX $switch" - AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], - [eval $cachevar=yes], - [eval $cachevar=no]) - CXX="$ac_save_CXX"]) - if eval test x\$$cachevar = xyes; then - CXX="$CXX $switch" - if test -n "$CXXCPP" ; then - CXXCPP="$CXXCPP $switch" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break fi - ac_success=yes + done + if test x$ac_success = xyes; then break fi done @@ -154,6 +148,11 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 ) +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) dnl Tests for new features in C++11 @@ -191,11 +190,13 @@ namespace cxx11 struct Base { + virtual ~Base() {} virtual void f() {} }; struct Derived : public Base { + virtual ~Derived() override {} virtual void f() override {} }; @@ -524,7 +525,7 @@ namespace cxx14 } - namespace test_digit_seperators + namespace test_digit_separators { constexpr auto ten_million = 100'000'000; @@ -566,3 +567,385 @@ namespace cxx14 #endif // __cplusplus >= 201402L ]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include <initializer_list> +#include <utility> +#include <type_traits> + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template<typename... Args> + int multiply(Args... args) + { + return (args * ... * 1); + } + + template<typename... Args> + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value); + static_assert(std::is_same<int, decltype(bar)>::value); + } + + namespace test_typename_in_template_template_parameter + { + + template<template<typename> typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template <bool cond> + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template <typename T1, typename T2> + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template <auto n> + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair<int, int> pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair<int, int>& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template<typename T> + Bad + f(T*, T*); + + template<typename T1, typename T2> + Good + f(T1*, T2*); + + static_assert (std::is_same_v<Good, decltype(f(g1, g2))>); + + } + + namespace test_inline_variables + { + + template<class T> void f(T) + {} + + template<class T> inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index c8b2fb1658..35ba8425b3 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -41,9 +41,6 @@ /* Define to 1 to enable ZMQ functions */ #define ENABLE_ZMQ 1 -/* parameter and return value type for __fdelt_chk */ -/* #undef FDELT_TYPE */ - /* define if the Boost library is available */ #define HAVE_BOOST /**/ diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index dae61c5e34..56b8fc976e 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -33,13 +33,15 @@ export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} -export TEST_PREVIOUS_RELEASES=${TEST_PREVIOUS_RELEASES:-false} +export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04} # Randomize test order. # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} +# See man 7 debconf +export DEBIAN_FRONTEND=noninteractive export CCACHE_SIZE=${CCACHE_SIZE:-100M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 43ee219ef9..f51cbf8f3c 100644 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -14,4 +14,4 @@ export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined --enable-c++17 CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index c27d525003..fb4c27c36f 100644 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -15,4 +15,4 @@ export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer --enable-c++17 CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh new file mode 100644 index 0000000000..0fc989085c --- /dev/null +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_multiprocess +export PACKAGES="cmake python3" +export DEP_OPTS="MULTIPROCESS=1" +export GOAL="install" +export BITCOIN_CONFIG="" +export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 87c8b70998..fa07990756 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -9,9 +9,9 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" -export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash +export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export TEST_PREVIOUS_RELEASES=true -export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" +export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.1 v0.18.1 v0.19.1" +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-c++17 --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 73ab5eebb6..63d06dea64 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -7,8 +7,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export DOCKER_NAME_TAG=ubuntu:16.04 -export PACKAGES="clang-8 llvm-8 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export DOCKER_NAME_TAG=ubuntu:18.04 +export PACKAGES="clang-9 llvm-9 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang-8 CXX=clang++-8" +export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang-9 CXX=clang++-9" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index d5b38a1d88..ff7c5fe913 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -10,10 +10,6 @@ export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -if [[ "${TRAVIS}" == "true" && "${TRAVIS_REPO_SLUG}" != "bitcoin/bitcoin" ]]; then - export TEST_RUNNER_EXTRA="wallet_disable" # Only run wallet_disable as a smoke test to not hit the 50 min travis time limit -else - export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 -fi +export TEST_RUNNER_EXTRA="--exclude rpc_bind --factor=2" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x_host.sh b/ci/test/00_setup_env_s390x_host.sh deleted file mode 100644 index 3e31867f2f..0000000000 --- a/ci/test/00_setup_env_s390x_host.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export HOST=s390x-linux-gnu -export NO_DEPENDS=1 -export BITCOIN_CONFIG="--with-incompatible-bdb --enable-reduce-exports" -export RUN_UNIT_TESTS=true -export RUN_FUNCTIONAL_TESTS=true -export GOAL="install" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 5dbf1b82f1..14ab943d68 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -26,7 +26,7 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order= export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" -env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|TEST_PREVIOUS_RELEASES|PREVIOUS_RELEASES_DIR)' | tee /tmp/env +env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN" elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index ab9d673101..3685504524 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -35,8 +35,8 @@ if [ -z "$NO_DEPENDS" ]; then fi DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS fi -if [ "$TEST_PREVIOUS_RELEASES" = "true" ]; then +if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then BEGIN_FOLD previous-versions - DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" v0.15.2 v0.16.3 v0.17.1 v0.18.1 v0.19.0.1 + DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" END_FOLD fi diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index c8542862f2..51d84ee39d 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -37,12 +37,12 @@ fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then BEGIN_FOLD functional-tests - DOCKER_EXEC test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast + DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast END_FOLD fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then BEGIN_FOLD fuzz-tests - DOCKER_EXEC test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} -l DEBUG ${DIR_FUZZ_IN} + DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN} END_FOLD fi diff --git a/configure.ac b/configure.ac index 473fe2064a..34b8354cda 100644 --- a/configure.ac +++ b/configure.ac @@ -61,8 +61,20 @@ case $host in lt_cv_deplibs_check_method="pass_all" ;; esac -dnl Require C++11 compiler (no GNU extensions) -AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault]) + +AC_ARG_ENABLE([c++17], + [AS_HELP_STRING([--enable-c++17], + [enable compilation in c++17 mode (disabled by default)])], + [use_cxx17=$enableval], + [use_cxx17=no]) + +dnl Require C++11 or C++17 compiler (no GNU extensions) +if test "x$use_cxx17" = xyes; then + AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) +else + AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory]) +fi + dnl Check if -latomic is required for <std::atomic> CHECK_ATOMIC @@ -230,6 +242,24 @@ if test x$enable_bip70 != xno; then AC_MSG_ERROR([BIP70 is no longer supported!]) fi +AC_ARG_WITH([libmultiprocess], + [AS_HELP_STRING([--with-libmultiprocess=yes|no|auto], + [Build with libmultiprocess library. (default: auto, i.e. detect with pkg-config)])], + [with_libmultiprocess=$withval], + [with_libmultiprocess=auto]) + +AC_ARG_WITH([mpgen], + [AS_HELP_STRING([--with-mpgen=yes|no|auto|PREFIX], + [Build with libmultiprocess codegen tool. Useful to specify different libmultiprocess host system library and build system codegen tool prefixes when cross-compiling (default is host system libmultiprocess prefix)])], + [with_mpgen=$withval], + [with_mpgen=auto]) + +AC_ARG_ENABLE([multiprocess], + [AS_HELP_STRING([--enable-multiprocess], + [build multiprocess bitcoin-node, bitcoin-wallet, and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables. Requires libmultiprocess library. Experimental (default is no)])], + [enable_multiprocess=$enableval], + [enable_multiprocess=no]) + AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], [do not install man pages (default is to install)])],, @@ -271,8 +301,30 @@ AC_ARG_ENABLE([werror], [enable_werror=no]) AC_LANG_PUSH([C++]) + +dnl Check for a flag to turn compiler warnings into errors. This is helpful for checks which may +dnl appear to succeed because by default they merely emit warnings when they fail. +dnl +dnl Note that this is not necessarily a check to see if -Werror is supported, but rather to see if +dnl a compile with -Werror can succeed. This is important because the compiler may already be +dnl warning about something unrelated, for example about some path issue. If that is the case, +dnl -Werror cannot be used because all of those warnings would be turned into errors. AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""]) +dnl Check for a flag to turn linker warnings into errors. When flags are passed to linkers via the +dnl compiler driver using a -Wl,-foo flag, linker warnings may be swallowed rather than bubbling up. +dnl See note above, the same applies here as well. +dnl +dnl LDFLAG_WERROR Should only be used when testing -Wl,* +case $host in + *darwin*) + AX_CHECK_LINK_FLAG([-Wl,-fatal_warnings],[LDFLAG_WERROR="-Wl,-fatal_warnings"],[LDFLAG_WERROR=""]) + ;; + *) + AX_CHECK_LINK_FLAG([-Wl,--fatal-warnings],[LDFLAG_WERROR="-Wl,--fatal-warnings"],[LDFLAG_WERROR=""]) + ;; +esac + if test "x$enable_debug" = xyes; then dnl Clear default -g -O2 flags if test "x$CXXFLAGS_overridden" = xno; then @@ -326,26 +378,38 @@ if test "x$enable_werror" = "xyes"; then if test "x$CXXFLAG_WERROR" = "x"; then AC_MSG_ERROR("enable-werror set but -Werror is not usable") fi + AX_CHECK_COMPILE_FLAG([-Werror=gnu],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=gnu"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=thread-safety-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety-analysis"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=thread-safety],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=date-time],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=date-time"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=return-type],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=return-type"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=conditional-uninitialized],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=conditional-uninitialized"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=sign-compare],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=sign-compare"],,[[$CXXFLAG_WERROR]]) + dnl -Wsuggest-override is broken with GCC before 9.2 + dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 + AX_CHECK_COMPILE_FLAG([-Werror=suggest-override],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=suggest-override"],,[[$CXXFLAG_WERROR]], + [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) fi if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wextra],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wgnu],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wswitch],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wswitch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat-security"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wthread-safety-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety-analysis"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wthread-safety],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-variable"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wsign-compare],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsign-compare"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], + [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all dnl unknown options if any other warning is produced. Test the -Wfoo case, and @@ -355,8 +419,12 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdeprecated-register],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-register"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"],,[[$CXXFLAG_WERROR]]) fi +dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. +AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers],[[CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"]],,[[$CXXFLAG_WERROR]]) + enable_sse42=no enable_sse41=no enable_avx2=no @@ -539,9 +607,6 @@ case $host in fi CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB -D_WIN32_WINNT=0x0601" - if test "x$CXXFLAGS_overridden" = "xno"; then - CXXFLAGS="$CXXFLAGS -w" - fi dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override @@ -551,6 +616,8 @@ case $host in archive_cmds_CXX="\$CC -shared \$libobjs \$deplibs \$compiler_flags -static -o \$output_objdir/\$soname \${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker \$lib" postdeps_CXX= + dnl We require Windows 7 (NT 6.1) or later + AX_CHECK_LINK_FLAG([[-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1]],[LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"],,[[$LDFLAG_WERROR]]) ;; *darwin*) TARGET_OS=darwin @@ -598,7 +665,7 @@ case $host in esac fi - AX_CHECK_LINK_FLAG([[-Wl,-headerpad_max_install_names]], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"]) + AX_CHECK_LINK_FLAG([[-Wl,-headerpad_max_install_names]], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"],, [[$LDFLAG_WERROR]]) CPPFLAGS="$CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" OBJCXXFLAGS="$CXXFLAGS" ;; @@ -680,20 +747,6 @@ AX_GCC_FUNC_ATTRIBUTE([dllexport]) AX_GCC_FUNC_ATTRIBUTE([dllimport]) if test x$use_glibc_compat != xno; then - - dnl __fdelt_chk's params and return type have changed from long unsigned int to long int. - dnl See which one is present here. - AC_MSG_CHECKING(__fdelt_chk type) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#ifdef _FORTIFY_SOURCE - #undef _FORTIFY_SOURCE - #endif - #define _FORTIFY_SOURCE 2 - #include <sys/select.h> - extern "C" long unsigned int __fdelt_warn(long unsigned int);]],[[]])], - [ fdelt_type="long unsigned int"], - [ fdelt_type="long int"]) - AC_MSG_RESULT($fdelt_type) - AC_DEFINE_UNQUOTED(FDELT_TYPE, $fdelt_type,[parameter and return value type for __fdelt_chk]) AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) else @@ -743,11 +796,11 @@ if test x$use_hardening != xno; then ]) fi - AX_CHECK_LINK_FLAG([[-Wl,--dynamicbase]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"]) - AX_CHECK_LINK_FLAG([[-Wl,--nxcompat]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"]) - AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"]) - AX_CHECK_LINK_FLAG([[-Wl,-z,relro]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"]) - AX_CHECK_LINK_FLAG([[-Wl,-z,now]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"]) + AX_CHECK_LINK_FLAG([[-Wl,--dynamicbase]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([[-Wl,--nxcompat]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([[-Wl,-z,relro]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([[-Wl,-z,now]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-fPIE -pie]], [PIE_FLAGS="-fPIE"; HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"],, [[$CXXFLAG_WERROR]]) case $host in @@ -761,52 +814,19 @@ dnl These flags are specific to ld64, and may cause issues with other linkers. dnl For example: GNU ld will intepret -dead_strip as -de and then try and use dnl "ad_strip" as the symbol for the entry point. if test x$TARGET_OS = xdarwin; then - AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"]) - AX_CHECK_LINK_FLAG([[-Wl,-dead_strip_dylibs]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"]) - AX_CHECK_LINK_FLAG([[-Wl,-bind_at_load]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"]) + AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([[-Wl,-dead_strip_dylibs]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([[-Wl,-bind_at_load]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"],, [[$LDFLAG_WERROR]]) fi if test x$enable_determinism = xyes; then if test x$TARGET_OS = xwindows; then - AX_CHECK_LINK_FLAG([[-Wl,--no-insert-timestamp]], [LDFLAGS="$LDFLAGS -Wl,--no-insert-timestamp"]) + AX_CHECK_LINK_FLAG([[-Wl,--no-insert-timestamp]], [LDFLAGS="$LDFLAGS -Wl,--no-insert-timestamp"],, [[$LDFLAG_WERROR]]) fi fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) -dnl FD_ZERO may be dependent on a declaration of memcpy, e.g. in SmartOS -dnl check that it fails to build without memcpy, then that it builds with -AC_MSG_CHECKING(FD_ZERO memcpy dependence) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include <cstddef> - #if HAVE_SYS_SELECT_H - #include <sys/select.h> - #endif - ]],[[ - #if HAVE_SYS_SELECT_H - fd_set fds; - FD_ZERO(&fds); - #endif - ]])], - [ AC_MSG_RESULT(no) ], - [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include <cstring> - #if HAVE_SYS_SELECT_H - #include <sys/select.h> - #endif - ]], [[ - #if HAVE_SYS_SELECT_H - fd_set fds; - FD_ZERO(&fds); - #endif - ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_CSTRING_DEPENDENT_FD_ZERO, 1, [Define this symbol if FD_ZERO is dependent of a memcpy declaration being available]) ], - [ AC_MSG_ERROR(failed with cstring include) ] - ) - ] -) - AC_CHECK_DECLS([getifaddrs, freeifaddrs],,, [#include <sys/types.h> #include <ifaddrs.h>] @@ -846,6 +866,22 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]], [ AC_MSG_RESULT(no)] ) +dnl Check for posix_fallocate +AC_MSG_CHECKING(for posix_fallocate) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + // same as in src/util/system.cpp + #ifdef __linux__ + #ifdef _POSIX_C_SOURCE + #undef _POSIX_C_SOURCE + #endif + #define _POSIX_C_SOURCE 200112L + #endif // __linux__ + #include <fcntl.h>]], + [[ int f = posix_fallocate(0, 0, 0); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_POSIX_FALLOCATE, 1,[Define this symbol if you have posix_fallocate]) ], + [ AC_MSG_RESULT(no)] +) + AC_MSG_CHECKING([for visibility attribute]) AC_LINK_IFELSE([AC_LANG_SOURCE([ int foo_def( void ) __attribute__((visibility("default"))); @@ -1184,7 +1220,7 @@ fi if test x$use_reduce_exports = xyes; then CXXFLAGS="$CXXFLAGS $RE_CXXFLAGS" - AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]], [RELDFLAGS="-Wl,--exclude-libs,ALL"]) + AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]], [RELDFLAGS="-Wl,--exclude-libs,ALL"],, [[$LDFLAG_WERROR]]) fi if test x$use_tests = xyes; then @@ -1374,6 +1410,50 @@ AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) AC_SUBST(UNIVALUE_CFLAGS) AC_SUBST(UNIVALUE_LIBS) +dnl libmultiprocess library check + +libmultiprocess_found=no +if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then + if test "x$use_pkgconfig" = xyes; then + m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ + libmultiprocess_found=yes; + libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; + ], [true])]) + fi +elif test "x$with_libmultiprocess" != xno; then + AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no]) +fi +AC_SUBST(LIBMULTIPROCESS_CFLAGS) +AC_SUBST(LIBMULTIPROCESS_LIBS) + +dnl Enable multiprocess check + +if test "x$enable_multiprocess" = xyes; then + if test "x$libmultiprocess_found" != xyes; then + AC_MSG_ERROR([--enable-multiprocess=yes option specified but libmultiprocess library was not found. May need to install libmultiprocess library, or specify install path with PKG_CONFIG_PATH environment variable. Running 'pkg-config --debug libmultiprocess' may be helpful for debugging.]) + fi + build_multiprocess=yes +elif test "x$enable_multiprocess" = xauto; then + build_multiprocess=$libmultiprocess_found +else + build_multiprocess=no +fi + +AM_CONDITIONAL([BUILD_MULTIPROCESS],[test "x$build_multiprocess" = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "x$build_multiprocess" = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "x$build_multiprocess" = xyes]) + +dnl codegen tools check + +if test x$build_multiprocess != xno; then + if test "x$with_mpgen" = xyes || test "x$with_mpgen" = xauto; then + MPGEN_PREFIX="$libmultiprocess_prefix" + elif test "x$with_mpgen" != xno; then + MPGEN_PREFIX="$with_mpgen"; + fi + AC_SUBST(MPGEN_PREFIX) +fi + AC_MSG_CHECKING([whether to build bitcoind]) AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes]) AC_MSG_RESULT($build_bitcoind) @@ -1415,9 +1495,6 @@ if test "x$use_ccache" != "xno"; then fi AC_MSG_RESULT($use_ccache) fi -if test "x$use_ccache" = "xyes"; then - AX_CHECK_PREPROC_FLAG([-Qunused-arguments],[CPPFLAGS="-Qunused-arguments $CPPFLAGS"]) -fi dnl enable wallet AC_MSG_CHECKING([if wallet should be enabled]) @@ -1655,6 +1732,7 @@ esac echo echo "Options used to compile and link:" +echo " multiprocess = $build_multiprocess" echo " with wallet = $enable_wallet" echo " with gui / qt = $bitcoin_enable_qt" if test x$bitcoin_enable_qt != xno; then diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index f5533719c0..bdff7a84b0 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -89,12 +89,6 @@ example: BUILDDIR=$PWD/build contrib/devtools/gen-manpages.sh ``` -optimize-pngs.py -================ - -A script to optimize png files in the bitcoin -repository (requires pngcrush). - security-check.py and test-security-check.py ============================================ diff --git a/contrib/devtools/optimize-pngs.py b/contrib/devtools/optimize-pngs.py deleted file mode 100755 index e9481dbbcf..0000000000 --- a/contrib/devtools/optimize-pngs.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Run this script every time you change one of the png files. Using pngcrush, it will optimize the png files, remove various color profiles, remove ancillary chunks (alla) and text chunks (text). -#pngcrush -brute -ow -rem gAMA -rem cHRM -rem iCCP -rem sRGB -rem alla -rem text -''' -import os -import sys -import subprocess -import hashlib -from PIL import Image # pip3 install Pillow - -def file_hash(filename): - '''Return hash of raw file contents''' - with open(filename, 'rb') as f: - return hashlib.sha256(f.read()).hexdigest() - -def content_hash(filename): - '''Return hash of RGBA contents of image''' - i = Image.open(filename) - i = i.convert('RGBA') - data = i.tobytes() - return hashlib.sha256(data).hexdigest() - -pngcrush = 'pngcrush' -git = 'git' -folders = ["src/qt/res/movies", "src/qt/res/icons", "share/pixmaps"] -basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel'], universal_newlines=True, encoding='utf8').rstrip('\n') -totalSaveBytes = 0 -noHashChange = True - -outputArray = [] -for folder in folders: - absFolder=os.path.join(basePath, folder) - for file in os.listdir(absFolder): - extension = os.path.splitext(file)[1] - if extension.lower() == '.png': - print("optimizing {}...".format(file), end =' ') - file_path = os.path.join(absFolder, file) - fileMetaMap = {'file' : file, 'osize': os.path.getsize(file_path), 'sha256Old' : file_hash(file_path)} - fileMetaMap['contentHashPre'] = content_hash(file_path) - - try: - subprocess.call([pngcrush, "-brute", "-ow", "-rem", "gAMA", "-rem", "cHRM", "-rem", "iCCP", "-rem", "sRGB", "-rem", "alla", "-rem", "text", file_path], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - except: - print("pngcrush is not installed, aborting...") - sys.exit(0) - - #verify - if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf8'): - print("PNG file "+file+" is corrupted after crushing, check out pngcursh version") - sys.exit(1) - - fileMetaMap['sha256New'] = file_hash(file_path) - fileMetaMap['contentHashPost'] = content_hash(file_path) - - if fileMetaMap['contentHashPre'] != fileMetaMap['contentHashPost']: - print("Image contents of PNG file {} before and after crushing don't match".format(file)) - sys.exit(1) - - fileMetaMap['psize'] = os.path.getsize(file_path) - outputArray.append(fileMetaMap) - print("done") - -print("summary:\n+++++++++++++++++") -for fileDict in outputArray: - oldHash = fileDict['sha256Old'] - newHash = fileDict['sha256New'] - totalSaveBytes += fileDict['osize'] - fileDict['psize'] - noHashChange = noHashChange and (oldHash == newHash) - print(fileDict['file']+"\n size diff from: "+str(fileDict['osize'])+" to: "+str(fileDict['psize'])+"\n old sha256: "+oldHash+"\n new sha256: "+newHash+"\n") - -print("completed. Checksum stable: "+str(noHashChange)+". Total reduction: "+str(totalSaveBytes)+" bytes") diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 9444271bdc..ca587ca9e5 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -12,33 +12,33 @@ import subprocess import sys import os +from typing import List, Optional + READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') -NONFATAL = {} # checks which are non-fatal for now but only generate a warning -def check_ELF_PIE(executable): +def run_command(command) -> str: + p = subprocess.run(command, stdout=subprocess.PIPE, check=True, universal_newlines=True) + return p.stdout + +def check_ELF_PIE(executable) -> bool: ''' Check for position independent executable (PIE), allowing for address space randomization. ''' - p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([READELF_CMD, '-h', '-W', executable]) ok = False for line in stdout.splitlines(): - line = line.split() - if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN': + tokens = line.split() + if len(line)>=2 and tokens[0] == 'Type:' and tokens[1] == 'DYN': ok = True return ok def get_ELF_program_headers(executable): '''Return type and flags for ELF program headers''' - p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([READELF_CMD, '-l', '-W', executable]) + in_headers = False count = 0 headers = [] @@ -62,7 +62,7 @@ def get_ELF_program_headers(executable): count += 1 return headers -def check_ELF_NX(executable): +def check_ELF_NX(executable) -> bool: ''' Check that no sections are writable and executable (including the stack) ''' @@ -75,7 +75,7 @@ def check_ELF_NX(executable): have_wx = True return have_gnu_stack and not have_wx -def check_ELF_RELRO(executable): +def check_ELF_RELRO(executable) -> bool: ''' Check for read-only relocations. GNU_RELRO program header must exist @@ -84,7 +84,8 @@ def check_ELF_RELRO(executable): have_gnu_relro = False for (typ, flags) in get_ELF_program_headers(executable): # Note: not checking flags == 'R': here as linkers set the permission differently - # This does not affect security: the permission flags of the GNU_RELRO program header are ignored, the PT_LOAD header determines the effective permissions. + # This does not affect security: the permission flags of the GNU_RELRO program + # header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also https://marc.info/?l=binutils&m=1498883354122353 @@ -92,93 +93,69 @@ def check_ELF_RELRO(executable): have_gnu_relro = True have_bindnow = False - p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([READELF_CMD, '-d', '-W', executable]) + for line in stdout.splitlines(): tokens = line.split() if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2:]): have_bindnow = True return have_gnu_relro and have_bindnow -def check_ELF_Canary(executable): +def check_ELF_Canary(executable) -> bool: ''' Check for use of stack canary ''' - p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([READELF_CMD, '--dyn-syms', '-W', executable]) + ok = False for line in stdout.splitlines(): if '__stack_chk_fail' in line: ok = True return ok -def get_PE_dll_characteristics(executable): - ''' - Get PE DllCharacteristics bits. - Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386' - and bits is the DllCharacteristics value. - ''' - p = subprocess.Popen([OBJDUMP_CMD, '-x', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') - arch = '' +def get_PE_dll_characteristics(executable) -> int: + '''Get PE DllCharacteristics bits''' + stdout = run_command([OBJDUMP_CMD, '-x', executable]) + bits = 0 for line in stdout.splitlines(): tokens = line.split() - if len(tokens)>=2 and tokens[0] == 'architecture:': - arch = tokens[1].rstrip(',') if len(tokens)>=2 and tokens[0] == 'DllCharacteristics': bits = int(tokens[1],16) - return (arch,bits) + return bits IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020 IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE = 0x0040 IMAGE_DLL_CHARACTERISTICS_NX_COMPAT = 0x0100 -def check_PE_DYNAMIC_BASE(executable): +def check_PE_DYNAMIC_BASE(executable) -> bool: '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - (arch,bits) = get_PE_dll_characteristics(executable) - reqbits = IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE - return (bits & reqbits) == reqbits + bits = get_PE_dll_characteristics(executable) + return (bits & IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE) == IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE -# On 64 bit, must support high-entropy 64-bit address space layout randomization in addition to DYNAMIC_BASE -# to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(executable): +# Must support high-entropy 64-bit address space layout randomization +# in addition to DYNAMIC_BASE to have secure ASLR. +def check_PE_HIGH_ENTROPY_VA(executable) -> bool: '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - (arch,bits) = get_PE_dll_characteristics(executable) - if arch == 'i386:x86-64': - reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA - else: # Unnecessary on 32-bit - assert(arch == 'i386') - reqbits = 0 - return (bits & reqbits) == reqbits + bits = get_PE_dll_characteristics(executable) + return (bits & IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA) == IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA def check_PE_RELOC_SECTION(executable) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' - p = subprocess.Popen([OBJDUMP_CMD, '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([OBJDUMP_CMD, '-h', executable]) + for line in stdout.splitlines(): if '.reloc' in line: return True return False -def check_PE_NX(executable): +def check_PE_NX(executable) -> bool: '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)''' - (arch,bits) = get_PE_dll_characteristics(executable) + bits = get_PE_dll_characteristics(executable) return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT -def get_MACHO_executable_flags(executable): - p = subprocess.Popen([OTOOL_CMD, '-vh', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') +def get_MACHO_executable_flags(executable) -> List[str]: + stdout = run_command([OTOOL_CMD, '-vh', executable]) flags = [] for line in stdout.splitlines(): @@ -222,10 +199,7 @@ def check_MACHO_LAZY_BINDINGS(executable) -> bool: Check for no lazy bindings. We don't use or check for MH_BINDATLOAD. See #18295. ''' - p = subprocess.Popen([OTOOL_CMD, '-l', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([OTOOL_CMD, '-l', executable]) for line in stdout.splitlines(): tokens = line.split() @@ -238,10 +212,8 @@ def check_MACHO_Canary(executable) -> bool: ''' Check for use of stack canary ''' - p = subprocess.Popen([OTOOL_CMD, '-Iv', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) - (stdout, stderr) = p.communicate() - if p.returncode: - raise IOError('Error opening file') + stdout = run_command([OTOOL_CMD, '-Iv', executable]) + ok = False for line in stdout.splitlines(): if '___stack_chk_fail' in line: @@ -270,7 +242,7 @@ CHECKS = { ] } -def identify_executable(executable): +def identify_executable(executable) -> Optional[str]: with open(filename, 'rb') as f: magic = f.read(4) if magic.startswith(b'MZ'): @@ -292,18 +264,12 @@ if __name__ == '__main__': continue failed = [] - warning = [] for (name, func) in CHECKS[etype]: if not func(filename): - if name in NONFATAL: - warning.append(name) - else: - failed.append(name) + failed.append(name) if failed: print('%s: failed %s' % (filename, ' '.join(failed))) retval = 1 - if warning: - print('%s: warning %s' % (filename, ' '.join(warning))) except IOError: print('%s: cannot open' % filename) retval = 1 diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME new file mode 100755 index 0000000000..a2ca768aaa --- /dev/null +++ b/contrib/gitian-descriptors/assign_DISTNAME @@ -0,0 +1,12 @@ +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# A helper script to be sourced into the gitian descriptors + +if RECENT_TAG="$(git describe --exact-match HEAD)"; then + VERSION="${RECENT_TAG#v}" +else + VERSION="$(git rev-parse --short=12 HEAD)" +fi +DISTNAME="bitcoin-${VERSION}" diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index f421372e10..0ed1e16f7e 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -140,9 +140,15 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. - # shellcheck source=contrib/gitian-descriptors/make_git_archive - source contrib/gitian-descriptors/make_git_archive + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME + + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" + + # Create the source tarball + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" # Extract the git archive into a dir for each host and build diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 82f8f194fc..bbae7201e5 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -103,9 +103,15 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. - # shellcheck source=contrib/gitian-descriptors/make_git_archive - source contrib/gitian-descriptors/make_git_archive + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME + + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" + + # Create the source tarball + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" # Extract the git archive into a dir for each host and build diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 54ad68a2a3..d05b6d426d 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -108,9 +108,15 @@ script: | create_per-host_compiler_wrapper "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. - # shellcheck source=contrib/gitian-descriptors/make_git_archive - source contrib/gitian-descriptors/make_git_archive + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME + + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" + + # Create the source tarball + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" # Extract the git archive into a dir for each host and build @@ -127,9 +133,8 @@ script: | make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make ${MAKEOPTS} -C src check-symbols - make deploy + make deploy BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" make install DESTDIR=${INSTALLPATH} - cp -f ./bitcoin-*-win64-setup-unsigned.exe ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete diff --git a/contrib/gitian-descriptors/make_git_archive b/contrib/gitian-descriptors/make_git_archive deleted file mode 100755 index d922c94c60..0000000000 --- a/contrib/gitian-descriptors/make_git_archive +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# A helper script to be sourced into the gitian descriptors - -mkdir -p ${OUTDIR}/src -RECENT_TAG=$(git describe --abbrev=0 HEAD) -if [ $RECENT_TAG = $(git describe HEAD) ]; then - if [[ $RECENT_TAG == v* ]]; then - VERSION=${RECENT_TAG:1} - else - VERSION=$RECENT_TAG - fi -else - VERSION=$(git rev-parse --short HEAD) -fi -DISTNAME=bitcoin-${VERSION} -GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" -git archive --output=$GIT_ARCHIVE HEAD diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 9f99b36f88..dffcf99607 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -13,7 +13,6 @@ We achieve bootstrappability by using Guix as a functional package manager. Conservatively, a x86_64 machine with: -- 2 or more logical cores - 4GB of free disk space on the partition that /gnu/store will reside in - 24GB of free disk space on the partition that the Bitcoin Core git repository resides in @@ -143,6 +142,11 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum If non-empty, will pass `V=1` to all `make` invocations, making `make` output verbose. + Note that any given value is ignored. The variable is only checked for + emptiness. More concretely, this means that `V=` (setting `V` to the empty + string) is interpreted the same way as not setting `V` at all, and that `V=0` + has the same effect as `V=1`. + * _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ Additional flags to be passed to `guix environment`. For a fully-bootstrapped diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index e20b2a048d..11d2c8b867 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -105,6 +105,7 @@ for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv --pure \ --no-cwd \ --share="$PWD"=/bitcoin \ + --expose="$(git rev-parse --git-common-dir)" \ ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 550b1b8f40..5be3baeefa 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -3,6 +3,14 @@ export LC_ALL=C set -e -o pipefail export TZ=UTC +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + # Check that environment variables assumed to be set by the environment are set echo "Building for platform triple ${HOST:?not set} with reference timestamp ${SOURCE_DATE_EPOCH:?not set}..." echo "At most ${MAX_JOBS:?not set} jobs will run at once..." @@ -141,19 +149,17 @@ make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ # Source Tarball Building # ########################### -# Create the source tarball and move it to "${OUTDIR}/src" if not already there -if [ -z "$(find "${OUTDIR}/src" -name 'bitcoin-*.tar.gz')" ]; then - ./autogen.sh - env CONFIG_SITE="${BASEPREFIX}/${HOST}/share/config.site" ./configure --prefix=/ - make dist GZIP_ENV='-9n' ${V:+V=1} - mkdir -p "${OUTDIR}/src" - mv "$(find "${PWD}" -name 'bitcoin-*.tar.gz')" "${OUTDIR}/src/" -fi +# Define DISTNAME variable. +# shellcheck source=contrib/gitian-descriptors/assign_DISTNAME +source contrib/gitian-descriptors/assign_DISTNAME + +GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" -# Determine the full path to our source tarball -SOURCEDIST="$(find "${OUTDIR}/src" -name 'bitcoin-*.tar.gz')" -# Determine our distribution name (e.g. bitcoin-0.18.0) -DISTNAME="$(basename "$SOURCEDIST" '.tar.gz')" +# Create the source tarball if not already there +if [ ! -e "$GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --output="$GIT_ARCHIVE" HEAD +fi ########################### # Binary Tarball Building # @@ -187,7 +193,9 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" cd "$DISTSRC" # Extract the source tarball - tar --strip-components=1 -xf "${SOURCEDIST}" + tar -xf "${GIT_ARCHIVE}" + + ./autogen.sh # Configure this DISTSRC for $HOST # shellcheck disable=SC2086 @@ -220,7 +228,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" # Make the os-specific installers case "$HOST" in *mingw*) - make deploy ${V:+V=1} + make deploy ${V:+V=1} BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" ;; esac @@ -232,11 +240,6 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" # Install built Bitcoin Core to $INSTALLPATH make install DESTDIR="${INSTALLPATH}" ${V:+V=1} - case "$HOST" in - *mingw*) - cp -f --target-directory="$OUTDIR" ./*-setup-unsigned.exe - ;; - esac ( cd installed @@ -264,7 +267,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" ;; *linux*) - cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + cp "${DISTSRC}/README.md" "${DISTNAME}/" ;; esac @@ -307,7 +310,7 @@ case "$HOST" in ( cd ./windeploy mkdir unsigned - cp --target-directory=unsigned/ "$OUTDIR"/bitcoin-*-setup-unsigned.exe + cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" find . -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 86c1a8d27f..5e011ea184 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -17,6 +17,7 @@ (gnu packages pkg-config) (gnu packages python) (gnu packages shells) + (gnu packages version-control) (guix build-system gnu) (guix build-system trivial) (guix gexp) @@ -181,6 +182,8 @@ chain for " target " development.")) ;; Scripting perl python-3.7 + ;; Git + git ;; Native gcc 9 toolchain targeting glibc 2.27 (make-gcc-toolchain gcc-9 glibc-2.27)) (let ((target (getenv "HOST"))) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 744b8ee70f..f232bb62c2 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -184,16 +184,3 @@ ... fun:_ZN5BCLog6Logger12StartLoggingEv } -{ - Suppress BCLog::Logger::StartLogging() still reachable memory warning - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - ... - fun:_ZN5BCLog6Logger12StartLoggingEv -} -{ - Suppress rest_blockhash_by_height Conditional jump or move depends on uninitialised value(s) - Memcheck:Cond - fun:_ZL24rest_blockhash_by_heightP11HTTPRequestRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE -} diff --git a/depends/Makefile b/depends/Makefile index 5f5247f881..5ad82bb56a 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -13,6 +13,7 @@ NO_QR ?= NO_WALLET ?= NO_ZMQ ?= NO_UPNP ?= +MULTIPROCESS ?= FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources BUILD = $(shell ./config.guess) @@ -107,6 +108,7 @@ qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch wallet_packages_$(NO_WALLET) = $(wallet_packages) upnp_packages_$(NO_UPNP) = $(upnp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) +multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) @@ -115,6 +117,11 @@ ifneq ($(zmq_packages_),) packages += $(zmq_packages) endif +ifeq ($(multiprocess_packages_),) +packages += $(multiprocess_packages) +native_packages += $(multiprocess_native_packages) +endif + all_packages = $(packages) $(native_packages) meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk @@ -155,6 +162,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@no_zmq@|$(NO_ZMQ)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ -e 's|@no_upnp@|$(NO_UPNP)|' \ + -e 's|@multiprocess@|$(MULTIPROCESS)|' \ -e 's|@debug@|$(DEBUG)|' \ $< > $@ $(AT)touch $@ diff --git a/depends/README.md b/depends/README.md index 79865ff011..c12ea8bcb3 100644 --- a/depends/README.md +++ b/depends/README.md @@ -91,6 +91,7 @@ The following can be set when running make: make FOO=bar NO_ZMQ: Don't download/build/cache packages needed for enabling zeromq NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp + MULTIPROCESS: build libmultiprocess (experimental, requires cmake) DEBUG: disable some optimizations and enable more runtime checking HOST_ID_SALT: Optional salt to use when generating host package ids BUILD_ID_SALT: Optional salt to use when generating build package ids diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index eb587fca89..69c394ec1d 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -1,5 +1,5 @@ -build_darwin_CC:=$(shell xcrun -f clang) -build_darwin_CXX:=$(shell xcrun -f clang++) +build_darwin_CC:=$(shell xcrun -f clang) --sysroot $(shell xcrun --show-sdk-path) +build_darwin_CXX:=$(shell xcrun -f clang++) --sysroot $(shell xcrun --show-sdk-path) build_darwin_AR:=$(shell xcrun -f ar) build_darwin_RANLIB:=$(shell xcrun -f ranlib) build_darwin_STRIP:=$(shell xcrun -f strip) diff --git a/depends/config.site.in b/depends/config.site.in index fb9bf713cc..103aa0d2ff 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -16,6 +16,9 @@ fi if test -z $with_qt_bindir && test -z "@no_qt@"; then with_qt_bindir=$depends_prefix/native/bin fi +if test -z $with_mpgen && test -n "@multiprocess@"; then + with_mpgen=$depends_prefix/native +fi if test -z $with_qrencode && test -n "@no_qr@"; then with_qrencode=no @@ -25,6 +28,10 @@ if test -z $enable_wallet && test -n "@no_wallet@"; then enable_wallet=no fi +if test -z $enable_multiprocess && test -n "@multiprocess@"; then + enable_multiprocess=yes +fi + if test -z $with_miniupnpc && test -n "@no_upnp@"; then with_miniupnpc=no fi diff --git a/depends/funcs.mk b/depends/funcs.mk index a4434b5167..135ebba9f8 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -130,11 +130,11 @@ $(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$( $(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig +$(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake $(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_stage_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" - ifneq ($($(1)_nm),) $(1)_autoconf += NM="$$($(1)_nm)" endif @@ -156,6 +156,15 @@ endif ifneq ($($(1)_ldflags),) $(1)_autoconf += LDFLAGS="$$($(1)_ldflags)" endif + +$(1)_cmake=cmake -DCMAKE_INSTALL_PREFIX=$($($(1)_type)_prefix) +ifneq ($($(1)_type),build) +ifneq ($(host),$(build)) +$(1)_cmake += -DCMAKE_SYSTEM_NAME=$($(host_os)_cmake_system) -DCMAKE_SYSROOT=$(host_prefix) +$(1)_cmake += -DCMAKE_C_COMPILER_TARGET=$(host) -DCMAKE_C_COMPILER=$(firstword $($($(1)_type)_CC)) -DCMAKE_C_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CC))" +$(1)_cmake += -DCMAKE_CXX_COMPILER_TARGET=$(host) -DCMAKE_CXX_COMPILER=$(firstword $($($(1)_type)_CXX)) -DCMAKE_CXX_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CXX))" +endif +endif endef define int_add_cmds diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk index 969ec2a1cb..eabd84bbbe 100644 --- a/depends/hosts/android.mk +++ b/depends/hosts/android.mk @@ -9,3 +9,4 @@ android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib endif +android_cmake_system=Android diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 1bc4fb8189..82e086a326 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -14,3 +14,4 @@ darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) darwin_native_toolchain=native_cctools +darwin_cmake_system=Darwin diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index b13a0f1ad7..8ab448ce5f 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -29,3 +29,4 @@ i686_linux_CXX=$(default_host_CXX) -m32 x86_64_linux_CC=$(default_host_CC) -m64 x86_64_linux_CXX=$(default_host_CXX) -m64 endif +linux_cmake_system=Linux diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index dbfb62fdcf..be5fec570c 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -8,3 +8,5 @@ mingw32_debug_CFLAGS=-O1 mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC + +mingw_cmake_system=Windows diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index cbe4fe4d97..970c81041e 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,7 +1,7 @@ package=boost $(package)_version=1_70_0 $(package)_download_path=https://dl.bintray.com/boostorg/release/1.70.0/source/ -$(package)_file_name=$(package)_$($(package)_version).tar.bz2 +$(package)_file_name=boost_$($(package)_version).tar.bz2 $(package)_sha256_hash=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 define $(package)_set_vars diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk new file mode 100644 index 0000000000..abeb26545f --- /dev/null +++ b/depends/packages/capnp.mk @@ -0,0 +1,18 @@ +package=capnp +$(package)_version=$(native_$(package)_version) +$(package)_download_path=$(native_$(package)_download_path) +$(package)_file_name=$(native_$(package)_file_name) +$(package)_sha256_hash=$(native_$(package)_sha256_hash) +$(package)_dependencies=native_$(package) + +define $(package)_config_cmds + $($(package)_autoconf) --with-external-capnp +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk new file mode 100644 index 0000000000..3e5cf5f160 --- /dev/null +++ b/depends/packages/libmultiprocess.mk @@ -0,0 +1,18 @@ +package=libmultiprocess +$(package)_version=$(native_$(package)_version) +$(package)_download_path=$(native_$(package)_download_path) +$(package)_file_name=$(native_$(package)_file_name) +$(package)_sha256_hash=$(native_$(package)_sha256_hash) +$(package)_dependencies=native_$(package) boost capnp + +define $(package)_config_cmds + $($(package)_cmake) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/native_capnp.mk b/depends/packages/native_capnp.mk new file mode 100644 index 0000000000..ed5a6deee2 --- /dev/null +++ b/depends/packages/native_capnp.mk @@ -0,0 +1,18 @@ +package=native_capnp +$(package)_version=0.7.0 +$(package)_download_path=https://capnproto.org/ +$(package)_download_file=capnproto-c++-$($(package)_version).tar.gz +$(package)_file_name=capnproto-cxx-$($(package)_version).tar.gz +$(package)_sha256_hash=c9a4c0bd88123064d483ab46ecee777f14d933359e23bff6fb4f4dbd28b4cd41 + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk new file mode 100644 index 0000000000..c50fdc3f6b --- /dev/null +++ b/depends/packages/native_libmultiprocess.mk @@ -0,0 +1,18 @@ +package=native_libmultiprocess +$(package)_version=5741d750a04e644a03336090d8979c6d033e32c0 +$(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=ac848db49a6ed53e423c62d54bd87f1f08cbb0326254a8667e10bbfe5bf032a4 +$(package)_dependencies=native_capnp + +define $(package)_config_cmds + $($(package)_cmake) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 42dbaa77a6..8fe2c771c9 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -16,6 +16,9 @@ zmq_packages=zeromq upnp_packages=miniupnpc +multiprocess_packages = libmultiprocess capnp +multiprocess_native_packages = native_libmultiprocess native_capnp + darwin_native_packages = native_biplist native_ds_store native_mac_alias ifneq ($(build_os),darwin) diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 366b1d0c42..631851855a 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -10,6 +10,7 @@ $(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch fix_android_qmake_conf.patch fix_android_jni_static.patch +# Update OSX_QT_TRANSLATIONS when this is updated $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=fb5a47799754af73d3bf501fe513342cfe2fc37f64e80df5533f6110e804220c diff --git a/doc/REST-interface.md b/doc/REST-interface.md index a63391e01a..11e9c90f49 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -30,7 +30,7 @@ To query for a confirmed transaction, enable the transaction index via "txindex= Given a block hash: returns a block, in binary, hex-encoded binary or JSON formats. Responds with 404 if the block doesn't exist. -The HTTP request and response are both handled entirely in-memory, thus making maximum memory usage at least 2.66MB (1 MB max block, plus hex encoding) per request. +The HTTP request and response are both handled entirely in-memory. With the /notxdetails/ option JSON response will only contain the transaction hash instead of the complete transaction details. The option only affects the JSON response. diff --git a/doc/build-windows.md b/doc/build-windows.md index bbff638b90..d3dc467f19 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -9,7 +9,7 @@ The options known to work for building Bitcoin Core on Windows are: and is the platform used to build the Bitcoin Core Windows release binaries. * On Windows, using [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and the Mingw-w64 cross compiler tool chain. -* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). +* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). See [README.md](/build_msvc/README.md). Other options which may work, but which have not been extensively tested are (please contribute instructions): diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 419b1db44e..16852bb970 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -8,7 +8,7 @@ To quickly get started fuzzing Bitcoin Core using [libFuzzer](https://llvm.org/d $ git clone https://github.com/bitcoin/bitcoin $ cd bitcoin/ $ ./autogen.sh -$ CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined +$ CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined --enable-c++17 # macOS users: If you have problem with this step then make sure to read "macOS hints for # libFuzzer" on https://github.com/bitcoin/bitcoin/blob/master/doc/fuzzing.md#macos-hints-for-libfuzzer $ make @@ -103,7 +103,7 @@ You may also need to take care of giving the correct path for `clang` and Full configure that was tested on macOS Catalina with `brew` installed `llvm`: ```sh -./configure --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ --disable-asm +./configure --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ --disable-asm --enable-c++17 ``` Read the [libFuzzer documentation](https://llvm.org/docs/LibFuzzer.html) for more information. This [libFuzzer tutorial](https://github.com/google/fuzzing/blob/master/tutorial/libFuzzerTutorial.md) might also be of interest. @@ -121,7 +121,7 @@ $ git clone https://github.com/google/afl $ make -C afl/ $ make -C afl/llvm_mode/ $ ./autogen.sh -$ CC=$(pwd)/afl/afl-clang-fast CXX=$(pwd)/afl/afl-clang-fast++ ./configure --enable-fuzz +$ CC=$(pwd)/afl/afl-clang-fast CXX=$(pwd)/afl/afl-clang-fast++ ./configure --enable-fuzz --enable-c++17 $ make # For macOS you may need to ignore x86 compilation checks when running "make". If so, # try compiling using: AFL_NO_X86=1 make @@ -148,7 +148,7 @@ $ git clone https://github.com/google/honggfuzz $ cd honggfuzz/ $ make $ cd .. -$ CC=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang CXX=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++ ./configure --enable-fuzz --with-sanitizers=address,undefined +$ CC=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang CXX=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++ ./configure --enable-fuzz --with-sanitizers=address,undefined --enable-c++17 $ make $ mkdir -p inputs/ $ honggfuzz/honggfuzz -i inputs/ -- src/test/fuzz/process_message diff --git a/doc/multiprocess.md b/doc/multiprocess.md new file mode 100644 index 0000000000..471d8561f7 --- /dev/null +++ b/doc/multiprocess.md @@ -0,0 +1,35 @@ +# Multiprocess Bitcoin + +On unix systems, the `--enable-multiprocess` build option can be passed to `./configure` to build new `bitcoin-node`, `bitcoin-wallet`, and `bitcoin-gui` executables alongside existing `bitcoind` and `bitcoin-qt` executables. + +`bitcoin-node` is a drop-in replacement for `bitcoind`, and `bitcoin-gui` is a drop-in replacement for `bitcoin-qt`, and there are no differences in use or external behavior between the new and old executables. But internally (after [#10102](https://github.com/bitcoin/bitcoin/pull/10102)), `bitcoin-gui` will spawn a `bitcoin-node` process to run P2P and RPC code, communicating with it across a socket pair, and `bitcoin-node` will spawn `bitcoin-wallet` to run wallet code, also communicating over a socket pair. This will let node, wallet, and GUI code run in separate address spaces for better isolation, and allow future improvements like being able to start and stop components independently on different machines and environments. + +## Next steps + +Specific next steps after [#10102](https://github.com/bitcoin/bitcoin/pull/10102) will be: + +- [ ] Adding `-ipcbind` and `-ipcconnect` options to `bitcoin-node`, `bitcoin-wallet`, and `bitcoin-gui` executables so they can listen and connect to TCP ports and unix socket paths. This will allow separate processes to be started and stopped any time and connect to each other. +- [ ] Adding `-server` and `-rpcbind` options to the `bitcoin-wallet` executable so wallet processes can handle RPC requests directly without going through the node. +- [ ] Supporting windows, not just unix systems. The existing socket code is already cross-platform, so the only windows-specific code that needs to be written is code spawning a process and passing a socket descriptor. This can be implemented with `CreateProcess` and `WSADuplicateSocket`. Example: https://memset.wordpress.com/2010/10/13/win32-api-passing-socket-with-ipc-method/. +- [ ] Adding sandbox features, restricting subprocess access to resources and data. See [https://eklitzke.org/multiprocess-bitcoin](https://eklitzke.org/multiprocess-bitcoin). + +## Debugging + +After [#10102](https://github.com/bitcoin/bitcoin/pull/10102), the `-debug=ipc` command line option can be used to see requests and responses between processes. + +## Installation + +The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) as dependencies. A simple way to get starting using it without installing these dependencies manually is to use the [depends system](../depends) with the `MULTIPROCESS=1` [dependency option](../depends#dependency-options) passed to make: + +``` +cd <BITCOIN_SOURCE_DIRECTORY> +make -C depends NO_QT=1 MULTIPROCESS=1 +./configure --prefix=$PWD/depends/x86_64-pc-linux-gnu +make +src/bitcoin-node -regtest -printtoconsole -debug=ipc +BITCOIND=bitcoin-node test/functional/test_runner.py +``` + +The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). + +Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. diff --git a/doc/productivity.md b/doc/productivity.md index 1bf3d9afb5..555f0afe3c 100644 --- a/doc/productivity.md +++ b/doc/productivity.md @@ -12,7 +12,7 @@ Table of Contents * [Multiple working directories with `git worktrees`](#multiple-working-directories-with-git-worktrees) * [Interactive "dummy rebases" for fixups and execs with `git merge-base`](#interactive-dummy-rebases-for-fixups-and-execs-with-git-merge-base) * [Writing code](#writing-code) - * [Format C/C++/Protobuf diffs with `clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy) + * [Format C/C++ diffs with `clang-format-diff.py`](#format-cc-diffs-with-clang-format-diffpy) * [Format Python diffs with `yapf-diff.py`](#format-python-diffs-with-yapf-diffpy) * [Rebasing/Merging code](#rebasingmerging-code) * [More conflict context with `merge.conflictstyle diff3`](#more-conflict-context-with-mergeconflictstyle-diff3) @@ -118,13 +118,13 @@ You can also set up [upstream refspecs](#reference-prs-easily-with-refspecs) to Writing code ------------ -### Format C/C++/Protobuf diffs with `clang-format-diff.py` +### Format C/C++ diffs with `clang-format-diff.py` See [contrib/devtools/README.md](/contrib/devtools/README.md#clang-format-diff.py). ### Format Python diffs with `yapf-diff.py` -Usage is exactly the same as [`clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy). You can get it [here](https://github.com/MarcoFalke/yapf-diff). +Usage is exactly the same as [`clang-format-diff.py`](#format-cc-diffs-with-clang-format-diffpy). You can get it [here](https://github.com/MarcoFalke/yapf-diff). Rebasing/Merging code ------------- diff --git a/doc/release-notes-16528.md b/doc/release-notes-16528.md new file mode 100644 index 0000000000..0a7a31cc9d --- /dev/null +++ b/doc/release-notes-16528.md @@ -0,0 +1,120 @@ +Wallet +------ + +### Experimental Descriptor Wallets + +Please note that Descriptor Wallets are still experimental and not all expected functionality +is available. Additionally there may be some bugs and current functions may change in the future. +Bugs and missing functionality can be reported to the [issue tracker](https://github.com/bitcoin/bitcoin/issues). + +0.21 introduces a new type of wallet - Descriptor Wallets. Descriptor Wallets store +scriptPubKey information using descriptors. This is in contrast to the Legacy Wallet +structure where keys are used to generate scriptPubKeys and addresses. Because of this +shift to being script based instead of key based, many of the confusing things that Legacy +Wallets do are not possible with Descriptor Wallets. Descriptor Wallets use a definition +of "mine" for scripts which is simpler and more intuitive than that used by Legacy Wallets. +Descriptor Wallets also uses different semantics for watch-only things and imports. + +As Descriptor Wallets are a new type of wallet, their introduction does not affect existing wallets. +Users who already have a Bitcoin Core wallet can continue to use it as they did before without +any change in behavior. Newly created Legacy Wallets (which is the default type of wallet) will +behave as they did in previous versions of Bitcoin Core. + +The differences between Descriptor Wallets and Legacy Wallets are largely limited to non user facing +things. They are intended to behave similarly except for the import/export and watchonly functionality +as described below. + +#### Creating Descriptor Wallets + +Descriptor Wallets are not created by default. They must be explicitly created using the +`createwallet` RPC or via the GUI. A `descriptors` option has been added to `createwallet`. +Setting `descriptors` to `true` will create a Descriptor Wallet instead of a Legacy Wallet. + +In the GUI, a checkbox has been added to the Create Wallet Dialog to indicate that a +Descriptor Wallet should be created. + +Without those options being set, a Legacy Wallet will be created instead. Additionally the +Default Wallet created upon first startup of Bitcoin Core will be a Legacy Wallet. + +#### `IsMine` Semantics + +`IsMine` refers to the function used to determine whether a script belongs to the wallet. +This is used to determine whether an output belongs to the wallet. `IsMine` in Legacy Wallets +returns true if the wallet would be able to sign an input that spends an output with that script. +Since keys can be involved in a variety of different scripts, this definition for `IsMine` can +lead to many unexpected scripts being considered part of the wallet. + +With Descriptor Wallets, descriptors explicitly specify the set of scripts that are owned by +the wallet. Since descriptors are deterministic and easily enumerable, users will know exactly +what scripts the wallet will consider to belong to it. Additionally the implementation of `IsMine` +in Descriptor Wallets is far simpler than for Legacy Wallets. Notably, in Legacy Wallets, `IsMine` +allowed for users to take one type of address (e.g. P2PKH), mutate it into another address type +(e.g. P2WPKH), and the wallet would still detect outputs sending to the new address type +even without that address being requested from the wallet. Descriptor Wallets does not +allow for this and will only watch for the addresses that were explicitly requested from the wallet. + +These changes to `IsMine` will make it easier to reason about what scripts the wallet will +actually be watching for in outputs. However for the vast majority of users, this change is +largely transparent and will not have noticeable effect. + +#### Imports and Exports + +In Legacy Wallets, raw scripts and keys could be imported to the wallet. Those imported scripts +and keys are treated separately from the keys generated by the wallet. This complicates the `IsMine` +logic as it has to distinguish between spendable and watchonly. + +Descriptor Wallets handle importing scripts and keys differently. Only complete descriptors can be +imported. These descriptors are then added to the wallet as if it were a descriptor generated by +the wallet itself. This simplifies the `IsMine` logic so that it no longer has to distinguish +between spendable and watchonly. As such, the watchonly model for Descriptor Wallets is also +different and described in more detail in the next section. + +To import into a Descriptor Wallet, a new `importdescriptors` RPC has been added that uses a syntax +similar to that of `importmulti`. + +As Legacy Wallets and Descriptor Wallets use different mechanisms for storing and importing scripts and keys +the existing import RPCs have been disabled for descriptor wallets. +New export RPCs for Descriptor Wallets have not yet been added. + +The following RPCs are disabled for Descriptor Wallets: + +* importprivkey +* importpubkey +* importaddress +* importwallet +* dumpprivkey +* dumpwallet +* importmulti +* addmultisigaddress +* sethdseed + +#### Watchonly Wallets + +A Legacy Wallet contains both private keys and scripts that were being watched. +Those watched scripts would not contribute to your normal balance. In order to see the watchonly +balance and to use watchonly things in transactions, an `include_watchonly` option was added +to many RPCs that would allow users to do that. However it is easy to forget to include this option. + +Descriptor Wallets move to a per-wallet watchonly model. Instead an entire wallet is considered to be +watchonly depending on whether it was created with private keys disabled. This eliminates the need +to distinguish between things that are watchonly and things that are not within a wallet itself. + +This change does have a caveat. If a Descriptor Wallet with private keys *enabled* has +a multiple key descriptor without all of the private keys (e.g. `multi(...)` with only one private key), +then the wallet will fail to sign and broadcast transactions. Such wallets would need to use the PSBT +workflow but the typical GUI Send, `sendtoaddress`, etc. workflows would still be available, just +non-functional. + +This issue is worsened if the wallet contains both single key (e.g. `wpkh(...)`) descriptors and such +multiple key descriptors as some transactions could be signed and broadast and others not. This is +due to some transactions containing only single key inputs, while others would contain both single +key and multiple key inputs, depending on which are available and how the coin selection algorithm +selects inputs. However this is not considered to be a supported use case; multisigs +should be in their own wallets which do not already have descriptors. Although users cannot export +descriptors with private keys for now as explained earlier. + +#### BIP 44/49/84 Support + +The change to using descriptors changes the default derivation paths used by Bitcoin Core +to adhere to BIP 44/49/84. Descriptors with different derivation paths can be imported without +issue. diff --git a/doc/release-notes-18918.md b/doc/release-notes-18918.md new file mode 100644 index 0000000000..f57a62eeb7 --- /dev/null +++ b/doc/release-notes-18918.md @@ -0,0 +1,3 @@ +# Wallet + +The `-salvagewallet` startup option has been removed. A new `salvage` command has been added to the `bitcoin-wallet` tool which performs the salvage operations that `-salvagewallet` did. diff --git a/doc/translation_strings_policy.md b/doc/translation_strings_policy.md index 634aca3559..1931302dda 100644 --- a/doc/translation_strings_policy.md +++ b/doc/translation_strings_policy.md @@ -23,7 +23,8 @@ On a high level, these strings are to be translated: ### GUI strings -Anything that appears to the user in the GUI is to be translated. This includes labels, menu items, button texts, tooltips and window titles. +Do not translate technical or extremely rare errors. +Anything else that appears to the user in the GUI is to be translated. This includes labels, menu items, button texts, tooltips and window titles. This includes messages passed to the GUI through the UI interface through `InitMessage`, `ThreadSafeMessageBox` or `ShowProgress`. General recommendations diff --git a/doc/zmq.md b/doc/zmq.md index a309abd0cc..3a1194de1c 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -96,7 +96,7 @@ ZeroMQ endpoint specifiers for TCP (and others) are documented in the Client side, then, the ZeroMQ subscriber socket must have the ZMQ_SUBSCRIBE option set to one or either of these prefixes (for instance, just `hash`); without doing so will result in no messages -arriving. Please see `contrib/zmq/zmq_sub.py` for a working example. +arriving. Please see [`contrib/zmq/zmq_sub.py`](/contrib/zmq/zmq_sub.py) for a working example. ## Remarks diff --git a/share/genbuild.sh b/share/genbuild.sh index 81fa2ed5d0..1153df933f 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -18,8 +18,8 @@ else exit 1 fi -DESC="" -SUFFIX="" +GIT_TAG="" +GIT_COMMIT="" if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ]; then # clean 'dirty' status of touched files that haven't been modified git diff >/dev/null 2>/dev/null @@ -27,18 +27,18 @@ if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$ # if latest commit is tagged and not dirty, then override using the tag name RAWDESC=$(git describe --abbrev=0 2>/dev/null) if [ "$(git rev-parse HEAD)" = "$(git rev-list -1 $RAWDESC 2>/dev/null)" ]; then - git diff-index --quiet HEAD -- && DESC=$RAWDESC + git diff-index --quiet HEAD -- && GIT_TAG=$RAWDESC fi # otherwise generate suffix from git, i.e. string like "59887e8-dirty" - SUFFIX=$(git rev-parse --short HEAD) - git diff-index --quiet HEAD -- || SUFFIX="$SUFFIX-dirty" + GIT_COMMIT=$(git rev-parse --short HEAD) + git diff-index --quiet HEAD -- || GIT_COMMIT="$GIT_COMMIT-dirty" fi -if [ -n "$DESC" ]; then - NEWINFO="#define BUILD_DESC \"$DESC\"" -elif [ -n "$SUFFIX" ]; then - NEWINFO="#define BUILD_SUFFIX $SUFFIX" +if [ -n "$GIT_TAG" ]; then + NEWINFO="#define BUILD_GIT_TAG \"$GIT_TAG\"" +elif [ -n "$GIT_COMMIT" ]; then + NEWINFO="#define BUILD_GIT_COMMIT \"$GIT_COMMIT\"" else NEWINFO="// No build information available" fi diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 4b2903a7c9..5431909bb2 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -51,7 +51,6 @@ Var StartMenuGroup !insertmacro MUI_LANGUAGE English # Installer attributes -OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-@PACKAGE_VERSION@-win64-setup-unsigned.exe InstallDir $PROGRAMFILES64\Bitcoin CRCCheck on XPStyle on diff --git a/src/Makefile.am b/src/Makefile.am index 627df97cad..2b004691fd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -85,6 +85,10 @@ if BUILD_BITCOIND bin_PROGRAMS += bitcoind endif +if BUILD_BITCOIN_NODE + bin_PROGRAMS += bitcoin-node +endif + if BUILD_BITCOIN_CLI bin_PROGRAMS += bitcoin-cli endif @@ -223,6 +227,7 @@ BITCOIN_CORE_H = \ util/message.h \ util/moneystr.h \ util/rbf.h \ + util/ref.h \ util/settings.h \ util/string.h \ util/threadnames.h \ @@ -243,6 +248,7 @@ BITCOIN_CORE_H = \ wallet/ismine.h \ wallet/load.h \ wallet/rpcwallet.h \ + wallet/salvage.h \ wallet/scriptpubkeyman.h \ wallet/wallet.h \ wallet/walletdb.h \ @@ -351,6 +357,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/load.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ + wallet/salvage.cpp \ wallet/scriptpubkeyman.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ @@ -496,7 +503,6 @@ libbitcoin_util_a_SOURCES = \ support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ - compat/glibc_sanity_fdelt.cpp \ compat/glibc_sanity.cpp \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ @@ -547,22 +553,21 @@ libbitcoin_cli_a_SOURCES = \ nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h # -# bitcoind binary # -bitcoind_SOURCES = bitcoind.cpp -bitcoind_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -bitcoind_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -bitcoind_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +# bitcoind & bitcoin-node binaries # +bitcoin_daemon_sources = bitcoind.cpp +bitcoin_bin_cppflags = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +bitcoin_bin_cxxflags = $(AM_CXXFLAGS) $(PIE_FLAGS) +bitcoin_bin_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) if TARGET_WINDOWS -bitcoind_SOURCES += bitcoind-res.rc +bitcoin_daemon_sources += bitcoind-res.rc endif -bitcoind_LDADD = \ - $(LIBBITCOIN_SERVER) \ +bitcoin_bin_ldadd = \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ - $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ + $(LIBUNIVALUE) \ $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ @@ -571,7 +576,19 @@ bitcoind_LDADD = \ $(LIBMEMENV) \ $(LIBSECP256K1) -bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) +bitcoin_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) + +bitcoind_SOURCES = $(bitcoin_daemon_sources) +bitcoind_CPPFLAGS = $(bitcoin_bin_cppflags) +bitcoind_CXXFLAGS = $(bitcoin_bin_cxxflags) +bitcoind_LDFLAGS = $(bitcoin_bin_ldflags) +bitcoind_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) + +bitcoin_node_SOURCES = $(bitcoin_daemon_sources) +bitcoin_node_CPPFLAGS = $(bitcoin_bin_cppflags) +bitcoin_node_CXXFLAGS = $(bitcoin_bin_cxxflags) +bitcoin_node_LDFLAGS = $(bitcoin_bin_ldflags) +bitcoin_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp @@ -615,29 +632,14 @@ bitcoin_tx_LDADD += $(BOOST_LIBS) # bitcoin-wallet binary # bitcoin_wallet_SOURCES = bitcoin-wallet.cpp -bitcoin_wallet_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -bitcoin_wallet_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -bitcoin_wallet_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +bitcoin_wallet_CPPFLAGS = $(bitcoin_bin_cppflags) +bitcoin_wallet_CXXFLAGS = $(bitcoin_bin_cxxflags) +bitcoin_wallet_LDFLAGS = $(bitcoin_bin_ldflags) +bitcoin_wallet_LDADD = $(LIBBITCOIN_WALLET_TOOL) $(bitcoin_bin_ldadd) if TARGET_WINDOWS bitcoin_wallet_SOURCES += bitcoin-wallet-res.rc endif - -bitcoin_wallet_LDADD = \ - $(LIBBITCOIN_WALLET_TOOL) \ - $(LIBBITCOIN_WALLET) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_ZMQ) \ - $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ - $(LIBMEMENV) \ - $(LIBSECP256K1) \ - $(LIBUNIVALUE) - -bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) # # bitcoinconsensus library # diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index 04b53471e4..8a28f4f249 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -36,7 +36,7 @@ LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX endif leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) -leveldb_libleveldb_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +leveldb_libleveldb_a_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized -Wsuggest-override -Werror=suggest-override, $(AM_CXXFLAGS)) $(PIE_FLAGS) leveldb_libleveldb_a_SOURCES= leveldb_libleveldb_a_SOURCES += leveldb/port/port_stdcxx.h diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index cf09eee2cb..13bfea7646 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -3,6 +3,11 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. bin_PROGRAMS += qt/bitcoin-qt + +if BUILD_BITCOIN_GUI + bin_PROGRAMS += bitcoin-gui +endif + EXTRA_LIBRARIES += qt/libbitcoinqt.a # bitcoin qt core # @@ -294,29 +299,43 @@ QT_FORMS_H=$(join $(dir $(QT_FORMS_UI)),$(addprefix ui_, $(notdir $(QT_FORMS_UI: # Most files will depend on the forms and moc files as includes. Generate them # before anything else. $(QT_MOC): $(QT_FORMS_H) -$(qt_libbitcoinqt_a_OBJECTS) $(qt_bitcoin_qt_OBJECTS) : | $(QT_MOC) +$(qt_libbitcoinqt_a_OBJECTS) $(qt_bitcoin_qt_OBJECTS) $(bitcoin_gui_OBJECTS) : | $(QT_MOC) -# bitcoin-qt binary # -qt_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ +# bitcoin-qt and bitcoin-gui binaries # +bitcoin_qt_cppflags = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(QR_CFLAGS) -qt_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) +bitcoin_qt_cxxflags = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) -qt_bitcoin_qt_SOURCES = qt/main.cpp +bitcoin_qt_sources = qt/main.cpp if TARGET_WINDOWS - qt_bitcoin_qt_SOURCES += $(BITCOIN_RC) + bitcoin_qt_sources += $(BITCOIN_RC) endif -qt_bitcoin_qt_LDADD = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) +bitcoin_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) if ENABLE_WALLET -qt_bitcoin_qt_LDADD += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) +bitcoin_qt_ldadd += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) endif if ENABLE_ZMQ -qt_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ +bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -qt_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -qt_bitcoin_qt_LIBTOOLFLAGS = $(AM_LIBTOOLFLAGS) --tag CXX +bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX + +qt_bitcoin_qt_CPPFLAGS = $(bitcoin_qt_cppflags) +qt_bitcoin_qt_CXXFLAGS = $(bitcoin_qt_cxxflags) +qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources) +qt_bitcoin_qt_LDADD = $(bitcoin_qt_ldadd) +qt_bitcoin_qt_LDFLAGS = $(bitcoin_qt_ldflags) +qt_bitcoin_qt_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) + +bitcoin_gui_CPPFLAGS = $(bitcoin_qt_cppflags) +bitcoin_gui_CXXFLAGS = $(bitcoin_qt_cxxflags) +bitcoin_gui_SOURCES = $(bitcoin_qt_sources) +bitcoin_gui_LDADD = $(bitcoin_qt_ldadd) +bitcoin_gui_LDFLAGS = $(bitcoin_qt_ldflags) +bitcoin_gui_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) #locale/foo.ts -> locale/foo.qm QT_QM=$(QT_TS:.ts=.qm) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 06dc59cf5c..2480cdadbb 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -9,6 +9,7 @@ FUZZ_TARGETS = \ test/fuzz/address_deserialize \ test/fuzz/addrman_deserialize \ test/fuzz/asmap \ + test/fuzz/asmap_direct \ test/fuzz/banentry_deserialize \ test/fuzz/base_encode_decode \ test/fuzz/bech32 \ @@ -30,6 +31,7 @@ FUZZ_TARGETS = \ test/fuzz/chain \ test/fuzz/checkqueue \ test/fuzz/coins_deserialize \ + test/fuzz/coins_view \ test/fuzz/cuckoocache \ test/fuzz/decode_tx \ test/fuzz/descriptor_parse \ @@ -49,9 +51,11 @@ FUZZ_TARGETS = \ test/fuzz/key \ test/fuzz/key_io \ test/fuzz/key_origin_info_deserialize \ + test/fuzz/kitchen_sink \ test/fuzz/locale \ test/fuzz/merkle_block_deserialize \ test/fuzz/merkleblock \ + test/fuzz/message \ test/fuzz/messageheader_deserialize \ test/fuzz/multiplication_overflow \ test/fuzz/net_permissions \ @@ -64,13 +68,13 @@ FUZZ_TARGETS = \ test/fuzz/parse_numbers \ test/fuzz/parse_script \ test/fuzz/parse_univalue \ - test/fuzz/prevector \ test/fuzz/partial_merkle_tree_deserialize \ test/fuzz/partially_signed_transaction_deserialize \ + test/fuzz/policy_estimator \ test/fuzz/pow \ test/fuzz/prefilled_transaction_deserialize \ + test/fuzz/prevector \ test/fuzz/primitives_transaction \ - test/fuzz/process_messages \ test/fuzz/process_message \ test/fuzz/process_message_addr \ test/fuzz/process_message_block \ @@ -96,12 +100,14 @@ FUZZ_TARGETS = \ test/fuzz/process_message_tx \ test/fuzz/process_message_verack \ test/fuzz/process_message_version \ + test/fuzz/process_messages \ test/fuzz/protocol \ test/fuzz/psbt \ test/fuzz/psbt_input_deserialize \ test/fuzz/psbt_output_deserialize \ test/fuzz/pub_key_deserialize \ test/fuzz/random \ + test/fuzz/rbf \ test/fuzz/rolling_bloom_filter \ test/fuzz/script \ test/fuzz/script_deserialize \ @@ -116,6 +122,7 @@ FUZZ_TARGETS = \ test/fuzz/string \ test/fuzz/strprintf \ test/fuzz/sub_net_deserialize \ + test/fuzz/system \ test/fuzz/timedata \ test/fuzz/transaction \ test/fuzz/tx_in \ @@ -223,6 +230,7 @@ BITCOIN_TESTS =\ test/prevector_tests.cpp \ test/raii_event_tests.cpp \ test/random_tests.cpp \ + test/ref_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ @@ -327,6 +335,12 @@ test_fuzz_asmap_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_asmap_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_asmap_SOURCES = test/fuzz/asmap.cpp +test_fuzz_asmap_direct_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_asmap_direct_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_asmap_direct_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_asmap_direct_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_asmap_direct_SOURCES = test/fuzz/asmap_direct.cpp + test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -453,6 +467,12 @@ test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_coins_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_coins_view_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_coins_view_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_coins_view_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_coins_view_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_coins_view_SOURCES = test/fuzz/coins_view.cpp + test_fuzz_cuckoocache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_cuckoocache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_cuckoocache_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -567,6 +587,12 @@ test_fuzz_key_origin_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_key_origin_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_key_origin_info_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_kitchen_sink_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_kitchen_sink_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_kitchen_sink_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_kitchen_sink_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_kitchen_sink_SOURCES = test/fuzz/kitchen_sink.cpp + test_fuzz_locale_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_locale_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_locale_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -585,6 +611,12 @@ test_fuzz_merkleblock_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_merkleblock_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_merkleblock_SOURCES = test/fuzz/merkleblock.cpp +test_fuzz_message_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_message_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_message_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_message_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_message_SOURCES = test/fuzz/message.cpp + test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -675,6 +707,12 @@ test_fuzz_partially_signed_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMO test_fuzz_partially_signed_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_partially_signed_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_policy_estimator_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_policy_estimator_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_policy_estimator_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_policy_estimator_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_policy_estimator_SOURCES = test/fuzz/policy_estimator.cpp + test_fuzz_pow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_pow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_pow_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -885,6 +923,12 @@ test_fuzz_random_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_random_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_random_SOURCES = test/fuzz/random.cpp +test_fuzz_rbf_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_rbf_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_rbf_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_rbf_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_rbf_SOURCES = test/fuzz/rbf.cpp + test_fuzz_rolling_bloom_filter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_rolling_bloom_filter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_rolling_bloom_filter_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -969,6 +1013,12 @@ test_fuzz_sub_net_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_sub_net_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_sub_net_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_system_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_system_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_system_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_system_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_system_SOURCES = test/fuzz/system.cpp + test_fuzz_timedata_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_timedata_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_timedata_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/addrman.cpp b/src/addrman.cpp index 02fb2fd491..7aba340d9d 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -644,5 +644,9 @@ std::vector<bool> CAddrMan::DecodeAsmap(fs::path path) bits.push_back((cur_byte >> bit) & 1); } } + if (!SanityCheckASMap(bits)) { + LogPrintf("Sanity check of asmap file %s failed\n", path); + return {}; + } return bits; } diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index d658976c3c..86f9a0bf67 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -47,8 +47,6 @@ static void CCoinsCaching(benchmark::State& state) while (state.KeepRunning()) { bool success = AreInputsStandard(tx_1, coins); assert(success); - CAmount value = coins.GetValueIn(tx_1); - assert(value == (50 + 21 + 22) * COIN); } ECC_Stop(); } diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index 00e5d7e7a0..42b351a72d 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -20,9 +20,7 @@ struct nontrivial_t { int x; nontrivial_t() :x(-1) {} - ADD_SERIALIZE_METHODS - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) {READWRITE(x);} + SERIALIZE_METHODS(nontrivial_t, obj) { READWRITE(obj.x); } }; static_assert(!IS_TRIVIALLY_CONSTRUCTIBLE<nontrivial_t>::value, "expected nontrivial_t to not be trivially constructible"); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index cdaabd6fab..8d85789b4e 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -9,6 +9,7 @@ #include <chainparamsbase.h> #include <clientversion.h> +#include <optional.h> #include <rpc/client.h> #include <rpc/protocol.h> #include <rpc/request.h> @@ -20,6 +21,7 @@ #include <functional> #include <memory> #include <stdio.h> +#include <string> #include <tuple> #include <event2/buffer.h> @@ -157,7 +159,7 @@ struct HTTPReply std::string body; }; -static const char *http_errorstring(int code) +static std::string http_errorstring(int code) { switch(code) { #if LIBEVENT_VERSION_NUMBER >= 0x02010300 @@ -250,7 +252,7 @@ public: UniValue ProcessReply(const UniValue &batch_in) override { UniValue result(UniValue::VOBJ); - std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in, batch_in.size()); + const std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in); // Errors in getnetworkinfo() and getblockchaininfo() are fatal, pass them on; // getwalletinfo() and getbalances() are allowed to fail if there is no wallet. if (!batch[ID_NETWORKINFO]["error"].isNull()) { @@ -304,7 +306,7 @@ public: } }; -static UniValue CallRPC(BaseRequestHandler *rh, const std::string& strMethod, const std::vector<std::string>& args) +static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, const std::vector<std::string>& args, const Optional<std::string>& rpcwallet = {}) { std::string host; // In preference order, we choose the following for the port: @@ -369,14 +371,12 @@ static UniValue CallRPC(BaseRequestHandler *rh, const std::string& strMethod, co // check if we should use a special wallet endpoint std::string endpoint = "/"; - if (!gArgs.GetArgs("-rpcwallet").empty()) { - std::string walletName = gArgs.GetArg("-rpcwallet", ""); - char *encodedURI = evhttp_uriencode(walletName.data(), walletName.size(), false); + if (rpcwallet) { + char* encodedURI = evhttp_uriencode(rpcwallet->data(), rpcwallet->size(), false); if (encodedURI) { - endpoint = "/wallet/"+ std::string(encodedURI); + endpoint = "/wallet/" + std::string(encodedURI); free(encodedURI); - } - else { + } else { throw CConnectionFailed("uri-encode failed"); } } @@ -418,6 +418,65 @@ static UniValue CallRPC(BaseRequestHandler *rh, const std::string& strMethod, co return reply; } +/** + * ConnectAndCallRPC wraps CallRPC with -rpcwait and an exception handler. + * + * @param[in] rh Pointer to RequestHandler. + * @param[in] strMethod Reference to const string method to forward to CallRPC. + * @param[in] rpcwallet Reference to const optional string wallet name to forward to CallRPC. + * @returns the RPC response as a UniValue object. + * @throws a CConnectionFailed std::runtime_error if connection failed or RPC server still in warmup. + */ +static UniValue ConnectAndCallRPC(BaseRequestHandler* rh, const std::string& strMethod, const std::vector<std::string>& args, const Optional<std::string>& rpcwallet = {}) +{ + UniValue response(UniValue::VOBJ); + // Execute and handle connection failures with -rpcwait. + const bool fWait = gArgs.GetBoolArg("-rpcwait", false); + do { + try { + response = CallRPC(rh, strMethod, args, rpcwallet); + if (fWait) { + const UniValue& error = find_value(response, "error"); + if (!error.isNull() && error["code"].get_int() == RPC_IN_WARMUP) { + throw CConnectionFailed("server in warmup"); + } + } + break; // Connection succeeded, no need to retry. + } catch (const CConnectionFailed&) { + if (fWait) { + UninterruptibleSleep(std::chrono::milliseconds{1000}); + } else { + throw; + } + } + } while (fWait); + return response; +} + +/** + * GetWalletBalances calls listwallets; if more than one wallet is loaded, it then + * fetches mine.trusted balances for each loaded wallet and pushes them to `result`. + * + * @param result Reference to UniValue object the wallet names and balances are pushed to. + */ +static void GetWalletBalances(UniValue& result) +{ + std::unique_ptr<BaseRequestHandler> rh{MakeUnique<DefaultRequestHandler>()}; + const UniValue listwallets = ConnectAndCallRPC(rh.get(), "listwallets", /* args=*/{}); + if (!find_value(listwallets, "error").isNull()) return; + const UniValue& wallets = find_value(listwallets, "result"); + if (wallets.size() <= 1) return; + + UniValue balances(UniValue::VOBJ); + for (const UniValue& wallet : wallets.getValues()) { + const std::string wallet_name = wallet.get_str(); + const UniValue getbalances = ConnectAndCallRPC(rh.get(), "getbalances", /* args=*/{}, wallet_name); + const UniValue& balance = find_value(getbalances, "result")["mine"]["trusted"]; + balances.pushKV(wallet_name, balance); + } + result.pushKV("balances", balances); +} + static int CommandLineRPC(int argc, char *argv[]) { std::string strPrint; @@ -474,9 +533,8 @@ static int CommandLineRPC(int argc, char *argv[]) } std::unique_ptr<BaseRequestHandler> rh; std::string method; - if (gArgs.GetBoolArg("-getinfo", false)) { + if (gArgs.IsArgSet("-getinfo")) { rh.reset(new GetinfoRequestHandler()); - method = ""; } else { rh.reset(new DefaultRequestHandler()); if (args.size() < 1) { @@ -485,62 +543,46 @@ static int CommandLineRPC(int argc, char *argv[]) method = args[0]; args.erase(args.begin()); // Remove trailing method name from arguments vector } - - // Execute and handle connection failures with -rpcwait - const bool fWait = gArgs.GetBoolArg("-rpcwait", false); - do { - try { - const UniValue reply = CallRPC(rh.get(), method, args); - - // Parse reply - const UniValue& result = find_value(reply, "result"); - const UniValue& error = find_value(reply, "error"); - - if (!error.isNull()) { - // Error - int code = error["code"].get_int(); - if (fWait && code == RPC_IN_WARMUP) - throw CConnectionFailed("server in warmup"); - strPrint = "error: " + error.write(); - nRet = abs(code); - if (error.isObject()) - { - UniValue errCode = find_value(error, "code"); - UniValue errMsg = find_value(error, "message"); - strPrint = errCode.isNull() ? "" : "error code: "+errCode.getValStr()+"\n"; - - if (errMsg.isStr()) - strPrint += "error message:\n"+errMsg.get_str(); - - if (errCode.isNum() && errCode.get_int() == RPC_WALLET_NOT_SPECIFIED) { - strPrint += "\nTry adding \"-rpcwallet=<filename>\" option to bitcoin-cli command line."; - } - } - } else { - // Result - if (result.isNull()) - strPrint = ""; - else if (result.isStr()) - strPrint = result.get_str(); - else - strPrint = result.write(2); + Optional<std::string> wallet_name{}; + if (gArgs.IsArgSet("-rpcwallet")) wallet_name = gArgs.GetArg("-rpcwallet", ""); + const UniValue reply = ConnectAndCallRPC(rh.get(), method, args, wallet_name); + + // Parse reply + UniValue result = find_value(reply, "result"); + const UniValue& error = find_value(reply, "error"); + if (!error.isNull()) { + // Error + strPrint = "error: " + error.write(); + nRet = abs(error["code"].get_int()); + if (error.isObject()) { + const UniValue& errCode = find_value(error, "code"); + const UniValue& errMsg = find_value(error, "message"); + strPrint = errCode.isNull() ? "" : ("error code: " + errCode.getValStr() + "\n"); + + if (errMsg.isStr()) { + strPrint += ("error message:\n" + errMsg.get_str()); + } + if (errCode.isNum() && errCode.get_int() == RPC_WALLET_NOT_SPECIFIED) { + strPrint += "\nTry adding \"-rpcwallet=<filename>\" option to bitcoin-cli command line."; } - // Connection succeeded, no need to retry. - break; } - catch (const CConnectionFailed&) { - if (fWait) - UninterruptibleSleep(std::chrono::milliseconds{1000}); - else - throw; + } else { + if (gArgs.IsArgSet("-getinfo") && !gArgs.IsArgSet("-rpcwallet")) { + GetWalletBalances(result); // fetch multiwallet balances and append to result } - } while (fWait); - } - catch (const std::exception& e) { + // Result + if (result.isNull()) { + strPrint = ""; + } else if (result.isStr()) { + strPrint = result.get_str(); + } else { + strPrint = result.write(2); + } + } + } catch (const std::exception& e) { strPrint = std::string("error: ") + e.what(); nRet = EXIT_FAILURE; - } - catch (...) { + } catch (...) { PrintExceptionContinue(nullptr, "CommandLineRPC()"); throw; } diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 7f9439788a..b420463c00 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -31,6 +31,7 @@ static void SetupWalletToolArgs() gArgs.AddArg("info", "Get wallet info", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); gArgs.AddArg("create", "Create new wallet file", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); + gArgs.AddArg("salvage", "Attempt to recover private keys from a corrupt wallet", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); } static bool WalletAppInit(int argc, char* argv[]) diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 2aa416ca44..b8e8717896 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -16,6 +16,7 @@ #include <noui.h> #include <shutdown.h> #include <ui_interface.h> +#include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> #include <util/threadnames.h> @@ -56,7 +57,7 @@ static bool AppInit(int argc, char* argv[]) SetupServerArgs(node); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { - return InitError(strprintf("Error parsing command line arguments: %s\n", error)); + return InitError(Untranslated(strprintf("Error parsing command line arguments: %s\n", error))); } // Process help and version before taking care about datadir @@ -77,25 +78,26 @@ static bool AppInit(int argc, char* argv[]) return true; } + util::Ref context{node}; try { if (!CheckDataDirOption()) { - return InitError(strprintf("Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", ""))); + return InitError(Untranslated(strprintf("Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")))); } if (!gArgs.ReadConfigFiles(error, true)) { - return InitError(strprintf("Error reading configuration file: %s\n", error)); + return InitError(Untranslated(strprintf("Error reading configuration file: %s\n", error))); } // Check for -chain, -testnet or -regtest parameter (Params() calls are only valid after this clause) try { SelectParams(gArgs.GetChainName()); } catch (const std::exception& e) { - return InitError(strprintf("%s\n", e.what())); + return InitError(Untranslated(strprintf("%s\n", e.what()))); } // Error out when loose non-argument tokens are encountered on command line for (int i = 1; i < argc; i++) { if (!IsSwitchChar(argv[i][0])) { - return InitError(strprintf("Command line contains unexpected token '%s', see bitcoind -h for a list of options.\n", argv[i])); + return InitError(Untranslated(strprintf("Command line contains unexpected token '%s', see bitcoind -h for a list of options.\n", argv[i]))); } } @@ -130,13 +132,13 @@ static bool AppInit(int argc, char* argv[]) // Daemonize if (daemon(1, 0)) { // don't chdir (1), do close FDs (0) - return InitError(strprintf("daemon() failed: %s\n", strerror(errno))); + return InitError(Untranslated(strprintf("daemon() failed: %s\n", strerror(errno)))); } #if defined(MAC_OSX) #pragma GCC diagnostic pop #endif #else - return InitError("-daemon is not supported on this operating system\n"); + return InitError(Untranslated("-daemon is not supported on this operating system\n")); #endif // HAVE_DECL_DAEMON } // Lock data directory after daemonization @@ -145,7 +147,7 @@ static bool AppInit(int argc, char* argv[]) // If locking the data directory failed, exit immediately return false; } - fRet = AppInitMain(node); + fRet = AppInitMain(context, node); } catch (const std::exception& e) { PrintExceptionContinue(&e, "AppInit()"); diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 263d863cfa..a47709cd82 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -105,13 +105,12 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c std::vector<bool> have_txn(txn_available.size()); { LOCK(pool->cs); - const std::vector<std::pair<uint256, CTxMemPool::txiter> >& vTxHashes = pool->vTxHashes; - for (size_t i = 0; i < vTxHashes.size(); i++) { - uint64_t shortid = cmpctblock.GetShortID(vTxHashes[i].first); + for (size_t i = 0; i < pool->vTxHashes.size(); i++) { + uint64_t shortid = cmpctblock.GetShortID(pool->vTxHashes[i].first); std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(shortid); if (idit != shorttxids.end()) { if (!have_txn[idit->second]) { - txn_available[idit->second] = vTxHashes[i].second->GetSharedTx(); + txn_available[idit->second] = pool->vTxHashes[i].second->GetSharedTx(); have_txn[idit->second] = true; mempool_count++; } else { diff --git a/src/blockencodings.h b/src/blockencodings.h index 377ac3a1a6..326db1b4a7 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -92,12 +92,13 @@ private: friend class PartiallyDownloadedBlock; - static const int SHORTTXIDS_LENGTH = 6; protected: std::vector<uint64_t> shorttxids; std::vector<PrefilledTransaction> prefilledtxn; public: + static constexpr int SHORTTXIDS_LENGTH = 6; + CBlockHeader header; // Dummy for deserialization @@ -125,7 +126,7 @@ class PartiallyDownloadedBlock { protected: std::vector<CTransactionRef> txn_available; size_t prefilled_count = 0, mempool_count = 0, extra_count = 0; - CTxMemPool* pool; + const CTxMemPool* pool; public: CBlockHeader header; explicit PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {} diff --git a/src/bloom.cpp b/src/bloom.cpp index 30af507243..54fcf487e4 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -31,8 +31,6 @@ CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, c * Again, we ignore filter parameters which will create a bloom filter with more hash functions than the protocol limits * See https://en.wikipedia.org/wiki/Bloom_filter for an explanation of these formulas */ - isFull(false), - isEmpty(true), nHashFuncs(std::min((unsigned int)(vData.size() * 8 / nElements * LN2), MAX_HASH_FUNCS)), nTweak(nTweakIn), nFlags(nFlagsIn) @@ -47,7 +45,7 @@ inline unsigned int CBloomFilter::Hash(unsigned int nHashNum, const std::vector< void CBloomFilter::insert(const std::vector<unsigned char>& vKey) { - if (isFull) + if (vData.empty()) // Avoid divide-by-zero (CVE-2013-5700) return; for (unsigned int i = 0; i < nHashFuncs; i++) { @@ -55,7 +53,6 @@ void CBloomFilter::insert(const std::vector<unsigned char>& vKey) // Sets bit nIndex of vData vData[nIndex >> 3] |= (1 << (7 & nIndex)); } - isEmpty = false; } void CBloomFilter::insert(const COutPoint& outpoint) @@ -74,10 +71,8 @@ void CBloomFilter::insert(const uint256& hash) bool CBloomFilter::contains(const std::vector<unsigned char>& vKey) const { - if (isFull) + if (vData.empty()) // Avoid divide-by-zero (CVE-2013-5700) return true; - if (isEmpty) - return false; for (unsigned int i = 0; i < nHashFuncs; i++) { unsigned int nIndex = Hash(i, vKey); @@ -112,10 +107,8 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) bool fFound = false; // Match if the filter contains the hash of tx // for finding tx when they appear in a block - if (isFull) + if (vData.empty()) // zero-size = "match-all" filter return true; - if (isEmpty) - return false; const uint256& hash = tx.GetHash(); if (contains(hash)) fFound = true; @@ -177,19 +170,6 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) return false; } -void CBloomFilter::UpdateEmptyFull() -{ - bool full = true; - bool empty = true; - for (unsigned int i = 0; i < vData.size(); i++) - { - full &= vData[i] == 0xff; - empty &= vData[i] == 0; - } - isFull = full; - isEmpty = empty; -} - CRollingBloomFilter::CRollingBloomFilter(const unsigned int nElements, const double fpRate) { double logFpRate = log(fpRate); diff --git a/src/bloom.h b/src/bloom.h index 8e3b7be54d..9307257852 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -45,8 +45,6 @@ class CBloomFilter { private: std::vector<unsigned char> vData; - bool isFull; - bool isEmpty; unsigned int nHashFuncs; unsigned int nTweak; unsigned char nFlags; @@ -64,17 +62,9 @@ public: * nFlags should be one of the BLOOM_UPDATE_* enums (not _MASK) */ CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak, unsigned char nFlagsIn); - CBloomFilter() : isFull(true), isEmpty(false), nHashFuncs(0), nTweak(0), nFlags(0) {} + CBloomFilter() : nHashFuncs(0), nTweak(0), nFlags(0) {} - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vData); - READWRITE(nHashFuncs); - READWRITE(nTweak); - READWRITE(nFlags); - } + SERIALIZE_METHODS(CBloomFilter, obj) { READWRITE(obj.vData, obj.nHashFuncs, obj.nTweak, obj.nFlags); } void insert(const std::vector<unsigned char>& vKey); void insert(const COutPoint& outpoint); @@ -90,9 +80,6 @@ public: //! Also adds any outputs which match the filter to the filter (to match their spending txes) bool IsRelevantAndUpdate(const CTransaction& tx); - - //! Checks for empty and full filters to avoid wasting cpu - void UpdateEmptyFull(); }; /** diff --git a/src/clientversion.cpp b/src/clientversion.cpp index 5d9eaea6d0..993967a180 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -14,59 +14,34 @@ */ const std::string CLIENT_NAME("Satoshi"); -/** - * Client version number - */ -#define CLIENT_VERSION_SUFFIX "" - - -/** - * The following part of the code determines the CLIENT_BUILD variable. - * Several mechanisms are used for this: - * * first, if HAVE_BUILD_INFO is defined, include build.h, a file that is - * generated by the build environment, possibly containing the output - * of git-describe in a macro called BUILD_DESC - * * secondly, if this is an exported version of the code, GIT_ARCHIVE will - * be defined (automatically using the export-subst git attribute), and - * GIT_COMMIT will contain the commit id. - * * then, three options exist for determining CLIENT_BUILD: - * * if BUILD_DESC is defined, use that literally (output of git-describe) - * * if not, but GIT_COMMIT is defined, use v[maj].[min].[rev].[build]-g[commit] - * * otherwise, use v[maj].[min].[rev].[build]-unk - * finally CLIENT_VERSION_SUFFIX is added - */ -//! First, include build.h if requested #ifdef HAVE_BUILD_INFO #include <obj/build.h> +// The <obj/build.h>, which is generated by the build environment (share/genbuild.sh), +// could contain only one line of the following: +// - "#define BUILD_GIT_TAG ...", if the top commit is tagged +// - "#define BUILD_GIT_COMMIT ...", if the top commit is not tagged +// - "// No build information available", if proper git information is not available #endif -//! git will put "#define GIT_ARCHIVE 1" on the next line inside archives. $Format:%n#define GIT_ARCHIVE 1$ -#ifdef GIT_ARCHIVE -#define GIT_COMMIT_ID "$Format:%H$" -#define GIT_COMMIT_DATE "$Format:%cD$" -#endif - -#define BUILD_DESC_WITH_SUFFIX(maj, min, rev, build, suffix) \ - "v" DO_STRINGIZE(maj) "." DO_STRINGIZE(min) "." DO_STRINGIZE(rev) "." DO_STRINGIZE(build) "-" DO_STRINGIZE(suffix) - -#define BUILD_DESC_FROM_COMMIT(maj, min, rev, build, commit) \ - "v" DO_STRINGIZE(maj) "." DO_STRINGIZE(min) "." DO_STRINGIZE(rev) "." DO_STRINGIZE(build) "-g" commit +//! git will put "#define GIT_COMMIT_ID ..." on the next line inside archives. $Format:%n#define GIT_COMMIT_ID "%H"$ -#define BUILD_DESC_FROM_UNKNOWN(maj, min, rev, build) \ - "v" DO_STRINGIZE(maj) "." DO_STRINGIZE(min) "." DO_STRINGIZE(rev) "." DO_STRINGIZE(build) "-unk" - -#ifndef BUILD_DESC -#ifdef BUILD_SUFFIX -#define BUILD_DESC BUILD_DESC_WITH_SUFFIX(CLIENT_VERSION_MAJOR, CLIENT_VERSION_MINOR, CLIENT_VERSION_REVISION, CLIENT_VERSION_BUILD, BUILD_SUFFIX) -#elif defined(GIT_COMMIT_ID) -#define BUILD_DESC BUILD_DESC_FROM_COMMIT(CLIENT_VERSION_MAJOR, CLIENT_VERSION_MINOR, CLIENT_VERSION_REVISION, CLIENT_VERSION_BUILD, GIT_COMMIT_ID) +#ifdef BUILD_GIT_TAG + #define BUILD_DESC BUILD_GIT_TAG + #define BUILD_SUFFIX "" #else -#define BUILD_DESC BUILD_DESC_FROM_UNKNOWN(CLIENT_VERSION_MAJOR, CLIENT_VERSION_MINOR, CLIENT_VERSION_REVISION, CLIENT_VERSION_BUILD) -#endif + #define BUILD_DESC "v" STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) \ + "." STRINGIZE(CLIENT_VERSION_REVISION) "." STRINGIZE(CLIENT_VERSION_BUILD) + #ifdef BUILD_GIT_COMMIT + #define BUILD_SUFFIX "-" BUILD_GIT_COMMIT + #elif defined(GIT_COMMIT_ID) + #define BUILD_SUFFIX "-g" GIT_COMMIT_ID + #else + #define BUILD_SUFFIX "-unk" + #endif #endif -const std::string CLIENT_BUILD(BUILD_DESC CLIENT_VERSION_SUFFIX); +const std::string CLIENT_BUILD(BUILD_DESC BUILD_SUFFIX); static std::string FormatVersion(int nVersion) { diff --git a/src/coins.cpp b/src/coins.cpp index 6b4cb2aec7..7b76c13f98 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -233,18 +233,6 @@ unsigned int CCoinsViewCache::GetCacheSize() const { return cacheCoins.size(); } -CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const -{ - if (tx.IsCoinBase()) - return 0; - - CAmount nResult = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - nResult += AccessCoin(tx.vin[i].prevout).out.nValue; - - return nResult; -} - bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const { if (!tx.IsCoinBase()) { diff --git a/src/coins.h b/src/coins.h index 2aed56c2bd..a3f34bb0ee 100644 --- a/src/coins.h +++ b/src/coins.h @@ -6,11 +6,11 @@ #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H -#include <primitives/transaction.h> #include <compressor.h> #include <core_memusage.h> #include <crypto/siphash.h> #include <memusage.h> +#include <primitives/transaction.h> #include <serialize.h> #include <uint256.h> @@ -25,7 +25,7 @@ * * Serialized format: * - VARINT((coinbase ? 1 : 0) | (height << 1)) - * - the non-spent CTxOut (via CTxOutCompressor) + * - the non-spent CTxOut (via TxOutCompression) */ class Coin { @@ -315,16 +315,6 @@ public: //! Calculate the size of the cache (in bytes) size_t DynamicMemoryUsage() const; - /** - * Amount of bitcoins coming in to a transaction - * Note that lightweight clients may not know anything besides the hash of previous transactions, - * so may not be able to calculate this. - * - * @param[in] tx transaction for which we are checking input total - * @return Sum of value of all inputs (scriptSigs) - */ - CAmount GetValueIn(const CTransaction& tx) const; - //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h index 6e7b4d3ded..4b0b224c69 100644 --- a/src/compat/assumptions.h +++ b/src/compat/assumptions.h @@ -50,6 +50,7 @@ static_assert(sizeof(double) == 8, "64-bit double assumed"); // code. static_assert(sizeof(short) == 2, "16-bit short assumed"); static_assert(sizeof(int) == 4, "32-bit int assumed"); +static_assert(sizeof(unsigned) == 4, "32-bit unsigned assumed"); // Assumption: We assume size_t to be 32-bit or 64-bit. // Example(s): size_t assumed to be at least 32-bit in ecdsa_signature_parse_der_lax(...). diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp index 4de4fd7f45..d17de33e86 100644 --- a/src/compat/glibc_compat.cpp +++ b/src/compat/glibc_compat.cpp @@ -9,10 +9,6 @@ #include <cstddef> #include <cstdint> -#if defined(HAVE_SYS_SELECT_H) -#include <sys/select.h> -#endif - // Prior to GLIBC_2.14, memcpy was aliased to memmove. extern "C" void* memmove(void* a, const void* b, size_t c); extern "C" void* memcpy(void* a, const void* b, size_t c) @@ -20,15 +16,6 @@ extern "C" void* memcpy(void* a, const void* b, size_t c) return memmove(a, b, c); } -extern "C" void __chk_fail(void) __attribute__((__noreturn__)); -extern "C" FDELT_TYPE __fdelt_warn(FDELT_TYPE a) -{ - if (a >= FD_SETSIZE) - __chk_fail(); - return a / __NFDBITS; -} -extern "C" FDELT_TYPE __fdelt_chk(FDELT_TYPE) __attribute__((weak, alias("__fdelt_warn"))); - #if defined(__i386__) || defined(__arm__) extern "C" int64_t __udivmoddi4(uint64_t u, uint64_t v, uint64_t* rp); diff --git a/src/compat/glibc_sanity.cpp b/src/compat/glibc_sanity.cpp index cc74f28899..0367b9a53f 100644 --- a/src/compat/glibc_sanity.cpp +++ b/src/compat/glibc_sanity.cpp @@ -8,10 +8,6 @@ #include <cstddef> -#if defined(HAVE_SYS_SELECT_H) -bool sanity_test_fdelt(); -#endif - extern "C" void* memcpy(void* a, const void* b, size_t c); void* memcpy_int(void* a, const void* b, size_t c) { @@ -45,9 +41,5 @@ bool sanity_test_memcpy() bool glibc_sanity_test() { -#if defined(HAVE_SYS_SELECT_H) - if (!sanity_test_fdelt()) - return false; -#endif return sanity_test_memcpy<1025>(); } diff --git a/src/compat/glibc_sanity_fdelt.cpp b/src/compat/glibc_sanity_fdelt.cpp deleted file mode 100644 index 87140d0c71..0000000000 --- a/src/compat/glibc_sanity_fdelt.cpp +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#if defined(HAVE_SYS_SELECT_H) -#ifdef HAVE_CSTRING_DEPENDENT_FD_ZERO -#include <cstring> -#endif -#include <sys/select.h> - -// trigger: Call FD_SET to trigger __fdelt_chk. FORTIFY_SOURCE must be defined -// as >0 and optimizations must be set to at least -O2. -// test: Add a file descriptor to an empty fd_set. Verify that it has been -// correctly added. -bool sanity_test_fdelt() -{ - fd_set fds; - FD_ZERO(&fds); - FD_SET(0, &fds); - return FD_ISSET(0, &fds); -} -#endif diff --git a/src/core_read.cpp b/src/core_read.cpp index df78c319ee..1c0a8a096d 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -19,6 +19,7 @@ #include <boost/algorithm/string/split.hpp> #include <algorithm> +#include <string> CScript ParseScript(const std::string& s) { @@ -34,10 +35,9 @@ CScript ParseScript(const std::string& s) if (op < OP_NOP && op != OP_RESERVED) continue; - const char* name = GetOpName(static_cast<opcodetype>(op)); - if (strcmp(name, "OP_UNKNOWN") == 0) + std::string strName = GetOpName(static_cast<opcodetype>(op)); + if (strName == "OP_UNKNOWN") continue; - std::string strName(name); mapOpNames[strName] = static_cast<opcodetype>(op); // Convenience: OP_ADD and just ADD are both recognized: boost::algorithm::replace_first(strName, "OP_", ""); diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index a5582e3b2c..0f7848bae1 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -8,6 +8,7 @@ class CWallet; enum class WalletCreationStatus; +struct bilingual_str; namespace interfaces { class Chain; @@ -72,12 +73,12 @@ std::vector<std::shared_ptr<CWallet>> GetWallets() throw std::logic_error("Wallet function called in non-wallet build."); } -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::vector<std::string>& warnings) +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) { throw std::logic_error("Wallet function called in non-wallet build."); } -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result) +WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result) { throw std::logic_error("Wallet function called in non-wallet build."); } diff --git a/src/flatfile.h b/src/flatfile.h index 60b3503cc3..04f6373a24 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -16,13 +16,7 @@ struct FlatFilePos int nFile; unsigned int nPos; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT_MODE(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nPos)); - } + SERIALIZE_METHODS(FlatFilePos, obj) { READWRITE(VARINT_MODE(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED), VARINT(obj.nPos)); } FlatFilePos() : nFile(-1), nPos(0) {} diff --git a/src/fs.cpp b/src/fs.cpp index 066c6c10d3..e68c97b3ca 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -6,6 +6,9 @@ #ifndef WIN32 #include <fcntl.h> +#include <string> +#include <sys/file.h> +#include <sys/utsname.h> #else #ifndef NOMINMAX #define NOMINMAX @@ -47,20 +50,38 @@ FileLock::~FileLock() } } +static bool IsWSL() +{ + struct utsname uname_data; + return uname(&uname_data) == 0 && std::string(uname_data.version).find("Microsoft") != std::string::npos; +} + bool FileLock::TryLock() { if (fd == -1) { return false; } - struct flock lock; - lock.l_type = F_WRLCK; - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; - if (fcntl(fd, F_SETLK, &lock) == -1) { - reason = GetErrorReason(); - return false; + + // Exclusive file locking is broken on WSL using fcntl (issue #18622) + // This workaround can be removed once the bug on WSL is fixed + static const bool is_wsl = IsWSL(); + if (is_wsl) { + if (flock(fd, LOCK_EX | LOCK_NB) == -1) { + reason = GetErrorReason(); + return false; + } + } else { + struct flock lock; + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + if (fcntl(fd, F_SETLK, &lock) == -1) { + reason = GetErrorReason(); + return false; + } } + return true; } #else diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 60c4d06f12..f1b9997371 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -9,7 +9,6 @@ #include <httpserver.h> #include <rpc/protocol.h> #include <rpc/server.h> -#include <ui_interface.h> #include <util/strencodings.h> #include <util/system.h> #include <util/translation.h> @@ -151,7 +150,7 @@ static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUserna return multiUserAuthorized(strUserPass); } -static bool HTTPReq_JSONRPC(HTTPRequest* req, const std::string &) +static bool HTTPReq_JSONRPC(const util::Ref& context, HTTPRequest* req) { // JSONRPC handles only POST if (req->GetRequestMethod() != HTTPRequest::POST) { @@ -166,7 +165,7 @@ static bool HTTPReq_JSONRPC(HTTPRequest* req, const std::string &) return false; } - JSONRPCRequest jreq; + JSONRPCRequest jreq(context); jreq.peerAddr = req->GetPeer().ToString(); if (!RPCAuthorized(authHeader.second, jreq.authUser)) { LogPrintf("ThreadRPCServer incorrect password attempt from %s\n", jreq.peerAddr); @@ -249,11 +248,8 @@ static bool InitRPCAuthentication() { if (gArgs.GetArg("-rpcpassword", "") == "") { - LogPrintf("No rpcpassword set - using random cookie authentication.\n"); + LogPrintf("Using random cookie authentication.\n"); if (!GenerateAuthCookie(&strRPCUserColonPass)) { - uiInterface.ThreadSafeMessageBox( - _("Error: A fatal internal error occurred, see debug.log for details").translated, // Same message as AbortNode - "", CClientUIInterface::MSG_ERROR); return false; } } else { @@ -288,15 +284,16 @@ static bool InitRPCAuthentication() return true; } -bool StartHTTPRPC() +bool StartHTTPRPC(const util::Ref& context) { LogPrint(BCLog::RPC, "Starting HTTP RPC server\n"); if (!InitRPCAuthentication()) return false; - RegisterHTTPHandler("/", true, HTTPReq_JSONRPC); + auto handle_rpc = [&context](HTTPRequest* req, const std::string&) { return HTTPReq_JSONRPC(context, req); }; + RegisterHTTPHandler("/", true, handle_rpc); if (g_wallet_init_interface.HasWalletSupport()) { - RegisterHTTPHandler("/wallet/", false, HTTPReq_JSONRPC); + RegisterHTTPHandler("/wallet/", false, handle_rpc); } struct event_base* eventBase = EventBase(); assert(eventBase); diff --git a/src/httprpc.h b/src/httprpc.h index 99e4d59b8a..a6a38fc95a 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -5,11 +5,14 @@ #ifndef BITCOIN_HTTPRPC_H #define BITCOIN_HTTPRPC_H +namespace util { +class Ref; +} // namespace util /** Start HTTP RPC subsystem. * Precondition; HTTP and RPC has been started. */ -bool StartHTTPRPC(); +bool StartHTTPRPC(const util::Ref& context); /** Interrupt HTTP RPC subsystem. */ void InterruptHTTPRPC(); @@ -21,7 +24,7 @@ void StopHTTPRPC(); /** Start HTTP REST subsystem. * Precondition; HTTP and RPC has been started. */ -void StartREST(); +void StartREST(const util::Ref& context); /** Interrupt RPC REST subsystem. */ void InterruptREST(); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 176284d103..5e78fd1d71 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -6,14 +6,15 @@ #include <chainparamsbase.h> #include <compat.h> -#include <util/threadnames.h> -#include <util/system.h> -#include <util/strencodings.h> #include <netbase.h> #include <rpc/protocol.h> // For HTTP status codes #include <shutdown.h> #include <sync.h> #include <ui_interface.h> +#include <util/strencodings.h> +#include <util/system.h> +#include <util/threadnames.h> +#include <util/translation.h> #include <deque> #include <memory> @@ -175,7 +176,7 @@ static bool InitHTTPAllowList() LookupSubNet(strAllow, subnet); if (!subnet.IsValid()) { uiInterface.ThreadSafeMessageBox( - strprintf("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24).", strAllow), + strprintf(Untranslated("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24)."), strAllow), "", CClientUIInterface::MSG_ERROR); return false; } @@ -420,7 +421,7 @@ bool UpdateHTTPServerLogging(bool enable) { #endif } -static std::thread threadHTTP; +static std::thread g_thread_http; static std::vector<std::thread> g_thread_http_workers; void StartHTTPServer() @@ -428,7 +429,7 @@ void StartHTTPServer() LogPrint(BCLog::HTTP, "Starting HTTP server\n"); int rpcThreads = std::max((long)gArgs.GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); - threadHTTP = std::thread(ThreadHTTP, eventBase); + g_thread_http = std::thread(ThreadHTTP, eventBase); for (int i = 0; i < rpcThreads; i++) { g_thread_http_workers.emplace_back(HTTPWorkQueueRun, workQueue, i); @@ -466,7 +467,7 @@ void StopHTTPServer() boundSockets.clear(); if (eventBase) { LogPrint(BCLog::HTTP, "Waiting for HTTP event thread to exit\n"); - threadHTTP.join(); + if (g_thread_http.joinable()) g_thread_http.join(); } if (eventHTTP) { evhttp_free(eventHTTP); diff --git a/src/index/base.cpp b/src/index/base.cpp index 7bff463f5b..74ea421e13 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -8,6 +8,7 @@ #include <tinyformat.h> #include <ui_interface.h> #include <util/system.h> +#include <util/translation.h> #include <validation.h> #include <warnings.h> @@ -23,7 +24,7 @@ static void FatalError(const char* fmt, const Args&... args) SetMiscWarning(strMessage); LogPrintf("*** %s\n", strMessage); uiInterface.ThreadSafeMessageBox( - "Error: A fatal internal error occurred, see debug.log for details", + Untranslated("Error: A fatal internal error occurred, see debug.log for details"), "", CClientUIInterface::MSG_ERROR); StartShutdown(); } diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index c3ce8d7af0..65a5f03a8e 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -31,6 +31,12 @@ constexpr char DB_FILTER_POS = 'P'; constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB /** The pre-allocation chunk size for fltr?????.dat files */ constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** Maximum size of the cfheaders cache + * We have a limit to prevent a bug in filling this cache + * potentially turning into an OOM. At 2000 entries, this cache + * is big enough for a 2,000,000 length block chain, which + * we should be enough until ~2047. */ +constexpr size_t CF_HEADERS_CACHE_MAX_SZ{2000}; namespace { @@ -39,14 +45,7 @@ struct DBVal { uint256 header; FlatFilePos pos; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(hash); - READWRITE(header); - READWRITE(pos); - } + SERIALIZE_METHODS(DBVal, obj) { READWRITE(obj.hash, obj.header, obj.pos); } }; struct DBHeightKey { @@ -78,17 +77,14 @@ struct DBHashKey { explicit DBHashKey(const uint256& hash_in) : hash(hash_in) {} - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(DBHashKey, obj) { char prefix = DB_BLOCK_HASH; READWRITE(prefix); if (prefix != DB_BLOCK_HASH) { throw std::ios_base::failure("Invalid format for block filter index DB hash key"); } - READWRITE(hash); + READWRITE(obj.hash); } }; @@ -387,13 +383,32 @@ bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& return ReadFilterFromDisk(entry.pos, filter_out); } -bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const +bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) { + LOCK(m_cs_headers_cache); + + bool is_checkpoint{block_index->nHeight % CFCHECKPT_INTERVAL == 0}; + + if (is_checkpoint) { + // Try to find the block in the headers cache if this is a checkpoint height. + auto header = m_headers_cache.find(block_index->GetBlockHash()); + if (header != m_headers_cache.end()) { + header_out = header->second; + return true; + } + } + DBVal entry; if (!LookupOne(*m_db, block_index, entry)) { return false; } + if (is_checkpoint && + m_headers_cache.size() < CF_HEADERS_CACHE_MAX_SZ) { + // Add to the headers cache if this is a checkpoint height. + m_headers_cache.emplace(block_index->GetBlockHash(), entry.header); + } + header_out = entry.header; return true; } diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index 436d52515f..317f8c0e40 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -10,6 +10,14 @@ #include <flatfile.h> #include <index/base.h> +/** Interval between compact filter checkpoints. See BIP 157. */ +static constexpr int CFCHECKPT_INTERVAL = 1000; + +struct FilterHeaderHasher +{ + size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } +}; + /** * BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of * blocks by height. An index is constructed for each supported filter type with its own database @@ -30,6 +38,10 @@ private: bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const; size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter); + Mutex m_cs_headers_cache; + /** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */ + std::unordered_map<uint256, uint256, FilterHeaderHasher> m_headers_cache GUARDED_BY(m_cs_headers_cache); + protected: bool Init() override; @@ -54,7 +66,7 @@ public: bool LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const; /** Get a single filter header by block. */ - bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const; + bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out); /** Get a range of filters between two heights on a chain. */ bool LookupFilterRange(int start_height, const CBlockIndex* stop_index, diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 5bbe6ad1df..4626395ef0 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -21,12 +21,10 @@ struct CDiskTxPos : public FlatFilePos { unsigned int nTxOffset; // after header - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(FlatFilePos, *this); - READWRITE(VARINT(nTxOffset)); + SERIALIZE_METHODS(CDiskTxPos, obj) + { + READWRITEAS(FlatFilePos, obj); + READWRITE(VARINT(obj.nTxOffset)); } CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { diff --git a/src/init.cpp b/src/init.cpp index a213dacbe0..37e6251295 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -59,9 +59,10 @@ #include <validationinterface.h> #include <walletinitinterface.h> +#include <functional> +#include <set> #include <stdint.h> #include <stdio.h> -#include <set> #ifndef WIN32 #include <attributes.h> @@ -121,7 +122,7 @@ NODISCARD static bool CreatePidFile() #endif return true; } else { - return InitError(strprintf(_("Unable to create the PID file '%s': %s").translated, GetPidFile().string(), std::strerror(errno))); + return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile().string(), std::strerror(errno))); } } @@ -243,9 +244,9 @@ void Shutdown(NodeContext& node) } // FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing - { + if (node.chainman) { LOCK(cs_main); - for (CChainState* chainstate : g_chainman.GetAll()) { + for (CChainState* chainstate : node.chainman->GetAll()) { if (chainstate->CanFlushToDisk()) { chainstate->ForceFlushStateToDisk(); } @@ -270,9 +271,9 @@ void Shutdown(NodeContext& node) // up with our current chain to avoid any strange pruning edge cases and make // next startup faster by avoiding rescan. - { + if (node.chainman) { LOCK(cs_main); - for (CChainState* chainstate : g_chainman.GetAll()) { + for (CChainState* chainstate : node.chainman->GetAll()) { if (chainstate->CanFlushToDisk()) { chainstate->ForceFlushStateToDisk(); chainstate->ResetCoinsViews(); @@ -298,7 +299,8 @@ void Shutdown(NodeContext& node) globalVerifyHandle.reset(); ECC_Stop(); node.args = nullptr; - if (node.mempool) node.mempool = nullptr; + node.mempool = nullptr; + node.chainman = nullptr; node.scheduler.reset(); try { @@ -350,13 +352,13 @@ static void registerSignalHandler(int signal, void(*handler)(int)) static boost::signals2::connection rpc_notify_block_change_connection; static void OnRPCStarted() { - rpc_notify_block_change_connection = uiInterface.NotifyBlockTip_connect(&RPCNotifyBlockChange); + rpc_notify_block_change_connection = uiInterface.NotifyBlockTip_connect(std::bind(RPCNotifyBlockChange, std::placeholders::_2)); } static void OnRPCStopped() { rpc_notify_block_change_connection.disconnect(); - RPCNotifyBlockChange(false, nullptr); + RPCNotifyBlockChange(nullptr); g_best_block_cv.notify_all(); LogPrint(BCLog::RPC, "RPC stopped.\n"); } @@ -446,6 +448,7 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor hidden services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-port=<port>", strprintf("Listen for connections on <port> (default: %u, testnet: %u, regtest: %u)", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -603,9 +606,9 @@ std::string LicenseInfo() } #if HAVE_SYSTEM -static void BlockNotifyCallback(bool initialSync, const CBlockIndex *pBlockIndex) +static void BlockNotifyCallback(SynchronizationState sync_state, const CBlockIndex* pBlockIndex) { - if (initialSync || !pBlockIndex) + if (sync_state != SynchronizationState::POST_INIT || !pBlockIndex) return; std::string strCmd = gArgs.GetArg("-blocknotify", ""); @@ -621,7 +624,7 @@ static bool fHaveGenesis = false; static Mutex g_genesis_wait_mutex; static std::condition_variable g_genesis_wait_cv; -static void BlockNotifyGenesisWait(bool, const CBlockIndex *pBlockIndex) +static void BlockNotifyGenesisWait(const CBlockIndex* pBlockIndex) { if (pBlockIndex != nullptr) { { @@ -687,7 +690,7 @@ static void CleanupBlockRevFiles() } } -static void ThreadImport(std::vector<fs::path> vImportFiles) +static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles) { const CChainParams& chainparams = Params(); util::ThreadRename("loadblk"); @@ -708,6 +711,10 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) break; // This error is logged in OpenBlockFile LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); LoadExternalBlockFile(chainparams, file, &pos); + if (ShutdownRequested()) { + LogPrintf("Shutdown requested. Exit %s\n", __func__); + return; + } nFile++; } pblocktree->WriteReindexing(false); @@ -723,6 +730,10 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) if (file) { LogPrintf("Importing blocks file %s...\n", path.string()); LoadExternalBlockFile(chainparams, file); + if (ShutdownRequested()) { + LogPrintf("Shutdown requested. Exit %s\n", __func__); + return; + } } else { LogPrintf("Warning: Could not open blocks file %s\n", path.string()); } @@ -731,9 +742,9 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) // scan for better chains in the block chain database, that are not yet connected in the active best chain // We can't hold cs_main during ActivateBestChain even though we're accessing - // the g_chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve + // the chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve // the relevant pointers before the ABC call. - for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + for (CChainState* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { BlockValidationState state; if (!chainstate->ActivateBestChain(state, chainparams, nullptr)) { LogPrintf("Failed to connect best block (%s)\n", state.ToString()); @@ -760,32 +771,30 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) */ static bool InitSanityCheck() { - if(!ECC_InitSanityCheck()) { - InitError("Elliptic curve cryptography sanity check failure. Aborting."); - return false; + if (!ECC_InitSanityCheck()) { + return InitError(Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")); } if (!glibc_sanity_test() || !glibcxx_sanity_test()) return false; if (!Random_SanityCheck()) { - InitError("OS cryptographic RNG sanity check failure. Aborting."); - return false; + return InitError(Untranslated("OS cryptographic RNG sanity check failure. Aborting.")); } return true; } -static bool AppInitServers() +static bool AppInitServers(const util::Ref& context) { RPCServer::OnStarted(&OnRPCStarted); RPCServer::OnStopped(&OnRPCStopped); if (!InitHTTPServer()) return false; StartRPC(); - if (!StartHTTPRPC()) + if (!StartHTTPRPC(context)) return false; - if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(); + if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(context); StartHTTPServer(); return true; } @@ -921,8 +930,9 @@ bool AppInitBasicSetup() HeapSetInformation(nullptr, HeapEnableTerminationOnCorruption, nullptr, 0); #endif - if (!SetupNetworking()) - return InitError("Initializing networking failed"); + if (!SetupNetworking()) { + return InitError(Untranslated("Initializing networking failed.")); + } #ifndef WIN32 if (!gArgs.GetBoolArg("-sysperms", false)) { @@ -959,16 +969,16 @@ bool AppInitParameterInteraction() // on the command line or in this network's section of the config file. std::string network = gArgs.GetChainName(); for (const auto& arg : gArgs.GetUnsuitableSectionOnlyArgs()) { - return InitError(strprintf(_("Config setting for %s only applied on %s network when in [%s] section.").translated, arg, network, network)); + return InitError(strprintf(_("Config setting for %s only applied on %s network when in [%s] section."), arg, network, network)); } // Warn if unrecognized section name are present in the config file. for (const auto& section : gArgs.GetUnrecognizedSections()) { - InitWarning(strprintf("%s:%i " + _("Section [%s] is not recognized.").translated, section.m_file, section.m_line, section.m_name)); + InitWarning(strprintf(Untranslated("%s:%i ") + _("Section [%s] is not recognized."), section.m_file, section.m_line, section.m_name)); } if (!fs::is_directory(GetBlocksDir())) { - return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist.").translated, gArgs.GetArg("-blocksdir", ""))); + return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", ""))); } // parse and validate enabled filter types @@ -980,25 +990,32 @@ bool AppInitParameterInteraction() for (const auto& name : names) { BlockFilterType filter_type; if (!BlockFilterTypeByName(name, filter_type)) { - return InitError(strprintf(_("Unknown -blockfilterindex value %s.").translated, name)); + return InitError(strprintf(_("Unknown -blockfilterindex value %s."), name)); } g_enabled_filter_types.insert(filter_type); } } + // Basic filters are the only supported filters. The basic filters index must be enabled + // to serve compact filters + if (gArgs.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS) && + g_enabled_filter_types.count(BlockFilterType::BASIC) != 1) { + return InitError(_("Cannot set -peerblockfilters without -blockfilterindex.")); + } + // if using block pruning, then disallow txindex if (gArgs.GetArg("-prune", 0)) { if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) - return InitError(_("Prune mode is incompatible with -txindex.").translated); + return InitError(_("Prune mode is incompatible with -txindex.")); if (!g_enabled_filter_types.empty()) { - return InitError(_("Prune mode is incompatible with -blockfilterindex.").translated); + return InitError(_("Prune mode is incompatible with -blockfilterindex.")); } } // -bind and -whitebind can't be set when not listening size_t nUserBind = gArgs.GetArgs("-bind").size() + gArgs.GetArgs("-whitebind").size(); if (nUserBind != 0 && !gArgs.GetBoolArg("-listen", DEFAULT_LISTEN)) { - return InitError("Cannot set -bind or -whitebind together with -listen=0"); + return InitError(Untranslated("Cannot set -bind or -whitebind together with -listen=0")); } // Make sure enough file descriptors are available @@ -1016,11 +1033,11 @@ bool AppInitParameterInteraction() #endif nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS), 0); if (nFD < MIN_CORE_FILEDESCRIPTORS) - return InitError(_("Not enough file descriptors available.").translated); + return InitError(_("Not enough file descriptors available.")); nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS, nMaxConnections); if (nMaxConnections < nUserMaxConnections) - InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations.").translated, nUserMaxConnections, nMaxConnections)); + InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags if (gArgs.IsArgSet("-debug")) { @@ -1031,7 +1048,7 @@ bool AppInitParameterInteraction() [](std::string cat){return cat == "0" || cat == "none";})) { for (const auto& cat : categories) { if (!LogInstance().EnableCategory(cat)) { - InitWarning(strprintf(_("Unsupported logging category %s=%s.").translated, "-debug", cat)); + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debug", cat)); } } } @@ -1040,7 +1057,7 @@ bool AppInitParameterInteraction() // Now remove the logging categories which were explicitly excluded for (const std::string& cat : gArgs.GetArgs("-debugexclude")) { if (!LogInstance().DisableCategory(cat)) { - InitWarning(strprintf(_("Unsupported logging category %s=%s.").translated, "-debugexclude", cat)); + InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); } } @@ -1061,7 +1078,7 @@ bool AppInitParameterInteraction() if (gArgs.IsArgSet("-minimumchainwork")) { const std::string minChainWorkStr = gArgs.GetArg("-minimumchainwork", ""); if (!IsHexNumber(minChainWorkStr)) { - return InitError(strprintf("Invalid non-hex (%s) minimum chain work value specified", minChainWorkStr)); + return InitError(strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), minChainWorkStr)); } nMinimumChainWork = UintToArith256(uint256S(minChainWorkStr)); } else { @@ -1076,21 +1093,21 @@ bool AppInitParameterInteraction() int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t nMempoolSizeMin = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; if (nMempoolSizeMax < 0 || nMempoolSizeMax < nMempoolSizeMin) - return InitError(strprintf(_("-maxmempool must be at least %d MB").translated, std::ceil(nMempoolSizeMin / 1000000.0))); + return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(nMempoolSizeMin / 1000000.0))); // incremental relay fee sets the minimum feerate increase necessary for BIP 125 replacement in the mempool // and the amount the mempool min fee increases above the feerate of txs evicted due to mempool limiting. if (gArgs.IsArgSet("-incrementalrelayfee")) { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-incrementalrelayfee", ""), n)) - return InitError(AmountErrMsg("incrementalrelayfee", gArgs.GetArg("-incrementalrelayfee", "")).translated); + return InitError(AmountErrMsg("incrementalrelayfee", gArgs.GetArg("-incrementalrelayfee", ""))); incrementalRelayFee = CFeeRate(n); } // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg = gArgs.GetArg("-prune", 0); if (nPruneArg < 0) { - return InitError(_("Prune cannot be configured with a negative value.").translated); + return InitError(_("Prune cannot be configured with a negative value.")); } nPruneTarget = (uint64_t) nPruneArg * 1024 * 1024; if (nPruneArg == 1) { // manual pruning: -prune=1 @@ -1099,7 +1116,7 @@ bool AppInitParameterInteraction() fPruneMode = true; } else if (nPruneTarget) { if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) { - return InitError(strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number.").translated, MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); + return InitError(strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); } LogPrintf("Prune configured to target %u MiB on disk for block and undo files.\n", nPruneTarget / 1024 / 1024); fPruneMode = true; @@ -1112,13 +1129,13 @@ bool AppInitParameterInteraction() peer_connect_timeout = gArgs.GetArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); if (peer_connect_timeout <= 0) { - return InitError("peertimeout cannot be configured with a negative value."); + return InitError(Untranslated("peertimeout cannot be configured with a negative value.")); } if (gArgs.IsArgSet("-minrelaytxfee")) { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-minrelaytxfee", ""), n)) { - return InitError(AmountErrMsg("minrelaytxfee", gArgs.GetArg("-minrelaytxfee", "")).translated); + return InitError(AmountErrMsg("minrelaytxfee", gArgs.GetArg("-minrelaytxfee", ""))); } // High fee check is done afterward in CWallet::CreateWalletFromFile() ::minRelayTxFee = CFeeRate(n); @@ -1134,7 +1151,7 @@ bool AppInitParameterInteraction() { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-blockmintxfee", ""), n)) - return InitError(AmountErrMsg("blockmintxfee", gArgs.GetArg("-blockmintxfee", "")).translated); + return InitError(AmountErrMsg("blockmintxfee", gArgs.GetArg("-blockmintxfee", ""))); } // Feerate used to define dust. Shouldn't be changed lightly as old @@ -1143,13 +1160,13 @@ bool AppInitParameterInteraction() { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-dustrelayfee", ""), n)) - return InitError(AmountErrMsg("dustrelayfee", gArgs.GetArg("-dustrelayfee", "")).translated); + return InitError(AmountErrMsg("dustrelayfee", gArgs.GetArg("-dustrelayfee", ""))); dustRelayFee = CFeeRate(n); } fRequireStandard = !gArgs.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (!chainparams.IsTestChain() && !fRequireStandard) { - return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); + return InitError(strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.NetworkIDString())); } nBytesPerSigOp = gArgs.GetArg("-bytespersigop", nBytesPerSigOp); @@ -1166,10 +1183,10 @@ bool AppInitParameterInteraction() nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); if (gArgs.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) - return InitError("rpcserialversion must be non-negative."); + return InitError(Untranslated("rpcserialversion must be non-negative.")); if (gArgs.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) - return InitError("unknown rpcserialversion requested."); + return InitError(Untranslated("Unknown rpcserialversion requested.")); nMaxTipAge = gArgs.GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); @@ -1181,10 +1198,10 @@ static bool LockDataDirectory(bool probeOnly) // Make sure only a single Bitcoin process is using the data directory. fs::path datadir = GetDataDir(); if (!DirIsWritable(datadir)) { - return InitError(strprintf(_("Cannot write to data directory '%s'; check permissions.").translated, datadir.string())); + return InitError(strprintf(_("Cannot write to data directory '%s'; check permissions."), datadir.string())); } if (!LockDirectory(datadir, ".lock", probeOnly)) { - return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running.").translated, datadir.string(), PACKAGE_NAME)); + return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), datadir.string(), PACKAGE_NAME)); } return true; } @@ -1202,7 +1219,7 @@ bool AppInitSanityChecks() // Sanity check if (!InitSanityCheck()) - return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down.").translated, PACKAGE_NAME)); + return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); // Probe the data directory lock to give an early error message, if possible // We cannot hold the data directory lock here, as the forking for daemon() hasn't yet happened, @@ -1222,7 +1239,7 @@ bool AppInitLockDataDirectory() return true; } -bool AppInitMain(NodeContext& node) +bool AppInitMain(const util::Ref& context, NodeContext& node) { const CChainParams& chainparams = Params(); // ********************************************************* Step 4a: application initialization @@ -1238,7 +1255,7 @@ bool AppInitMain(NodeContext& node) } } if (!LogInstance().StartLogging()) { - return InitError(strprintf("Could not open debug log file %s", + return InitError(strprintf(Untranslated("Could not open debug log file %s"), LogInstance().m_file_path.string())); } @@ -1253,7 +1270,7 @@ bool AppInitMain(NodeContext& node) LogPrintf("Config file: %s\n", config_file_path.string()); } else if (gArgs.IsArgSet("-conf")) { // Warn if no conf file exists at path provided by user - InitWarning(strprintf(_("The specified config file %s does not exist\n").translated, config_file_path.string())); + InitWarning(strprintf(_("The specified config file %s does not exist\n"), config_file_path.string())); } else { // Not categorizing as "Warning" because it's the default behavior LogPrintf("Config file: %s (not found, skipping)\n", config_file_path.string()); @@ -1324,7 +1341,6 @@ bool AppInitMain(NodeContext& node) for (const auto& client : node.chain_clients) { client->registerRpcs(); } - g_rpc_node = &node; #if ENABLE_ZMQ RegisterZMQRPCCommands(tableRPC); #endif @@ -1337,8 +1353,8 @@ bool AppInitMain(NodeContext& node) if (gArgs.GetBoolArg("-server", false)) { uiInterface.InitMessage_connect(SetRPCWarmupStatus); - if (!AppInitServers()) - return InitError(_("Unable to start HTTP server. See debug log for details.").translated); + if (!AppInitServers(context)) + return InitError(_("Unable to start HTTP server. See debug log for details.")); } // ********************************************************* Step 5: verify wallet database integrity @@ -1362,20 +1378,23 @@ bool AppInitMain(NodeContext& node) // which are all started after this, may use it from the node context. assert(!node.mempool); node.mempool = &::mempool; + assert(!node.chainman); + node.chainman = &g_chainman; + ChainstateManager& chainman = EnsureChainman(node); - node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, *node.mempool)); + node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, *node.chainman, *node.mempool)); RegisterValidationInterface(node.peer_logic.get()); // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; for (const std::string& cmt : gArgs.GetArgs("-uacomment")) { if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) - return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters.").translated, cmt)); + return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters."), cmt)); uacomments.push_back(cmt); } strSubVersion = FormatSubVersion(CLIENT_NAME, CLIENT_VERSION, uacomments); if (strSubVersion.size() > MAX_SUBVERSION_LENGTH) { - return InitError(strprintf(_("Total length of network version string (%i) exceeds maximum length (%i). Reduce the number or size of uacomments.").translated, + return InitError(strprintf(_("Total length of network version string (%i) exceeds maximum length (%i). Reduce the number or size of uacomments."), strSubVersion.size(), MAX_SUBVERSION_LENGTH)); } @@ -1384,7 +1403,7 @@ bool AppInitMain(NodeContext& node) for (const std::string& snet : gArgs.GetArgs("-onlynet")) { enum Network net = ParseNetwork(snet); if (net == NET_UNROUTABLE) - return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'").translated, snet)); + return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'"), snet)); nets.insert(net); } for (int n = 0; n < NET_MAX; n++) { @@ -1405,12 +1424,12 @@ bool AppInitMain(NodeContext& node) if (proxyArg != "" && proxyArg != "0") { CService proxyAddr; if (!Lookup(proxyArg, proxyAddr, 9050, fNameLookup)) { - return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'").translated, proxyArg)); + return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); } proxyType addrProxy = proxyType(proxyAddr, proxyRandomize); if (!addrProxy.IsValid()) - return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'").translated, proxyArg)); + return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); SetProxy(NET_IPV4, addrProxy); SetProxy(NET_IPV6, addrProxy); @@ -1429,11 +1448,11 @@ bool AppInitMain(NodeContext& node) } else { CService onionProxy; if (!Lookup(onionArg, onionProxy, 9050, fNameLookup)) { - return InitError(strprintf(_("Invalid -onion address or hostname: '%s'").translated, onionArg)); + return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); } proxyType addrOnion = proxyType(onionProxy, proxyRandomize); if (!addrOnion.IsValid()) - return InitError(strprintf(_("Invalid -onion address or hostname: '%s'").translated, onionArg)); + return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); SetProxy(NET_ONION, addrOnion); SetReachable(NET_ONION, true); } @@ -1449,7 +1468,7 @@ bool AppInitMain(NodeContext& node) if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); else - return InitError(ResolveErrMsg("externalip", strAddr)); + return InitError(Untranslated(ResolveErrMsg("externalip", strAddr))); } // Read asmap file if configured @@ -1462,12 +1481,12 @@ bool AppInitMain(NodeContext& node) asmap_path = GetDataDir() / asmap_path; } if (!fs::exists(asmap_path)) { - InitError(strprintf(_("Could not find asmap file %s").translated, asmap_path)); + InitError(strprintf(_("Could not find asmap file %s"), asmap_path)); return false; } std::vector<bool> asmap = CAddrMan::DecodeAsmap(asmap_path); if (asmap.size() == 0) { - InitError(strprintf(_("Could not parse asmap file %s").translated, asmap_path)); + InitError(strprintf(_("Could not parse asmap file %s"), asmap_path)); return false; } const uint256 asmap_version = SerializeHash(asmap); @@ -1534,7 +1553,7 @@ bool AppInitMain(NodeContext& node) auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); }; - std::string strLoadError; + bilingual_str strLoadError; uiInterface.InitMessage(_("Loading block index...").translated); @@ -1542,7 +1561,7 @@ bool AppInitMain(NodeContext& node) const int64_t load_block_index_start_time = GetTimeMillis(); try { LOCK(cs_main); - g_chainman.InitializeChainstate(); + chainman.InitializeChainstate(); UnloadBlockIndex(); // new CBlockTreeDB tries to delete the existing file, which @@ -1563,9 +1582,9 @@ bool AppInitMain(NodeContext& node) // block file from disk. // Note that it also sets fReindex based on the disk flag! // From here on out fReindex and fReset mean something different! - if (!LoadBlockIndex(chainparams)) { + if (!chainman.LoadBlockIndex(chainparams)) { if (ShutdownRequested()) break; - strLoadError = _("Error loading block database").translated; + strLoadError = _("Error loading block database"); break; } @@ -1573,13 +1592,13 @@ bool AppInitMain(NodeContext& node) // (we're likely using a testnet datadir, or the other way around). if (!::BlockIndex().empty() && !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { - return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?").translated); + return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); } // Check for changed -prune state. What we are concerned about is a user who has pruned blocks // in the past, but is now trying to run unpruned. if (fHavePruned && !fPruneMode) { - strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain").translated; + strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain"); break; } @@ -1588,7 +1607,7 @@ bool AppInitMain(NodeContext& node) // (otherwise we use the one already on disk). // This is called again in ThreadImport after the reindex completes. if (!fReindex && !LoadGenesisBlock(chainparams)) { - strLoadError = _("Error initializing block database").translated; + strLoadError = _("Error initializing block database"); break; } @@ -1597,7 +1616,7 @@ bool AppInitMain(NodeContext& node) bool failed_chainstate_init = false; - for (CChainState* chainstate : g_chainman.GetAll()) { + for (CChainState* chainstate : chainman.GetAll()) { LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); chainstate->InitCoinsDB( /* cache_size_bytes */ nCoinDBCache, @@ -1606,21 +1625,21 @@ bool AppInitMain(NodeContext& node) chainstate->CoinsErrorCatcher().AddReadErrCallback([]() { uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down.").translated, + _("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); }); // If necessary, upgrade from older database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (!chainstate->CoinsDB().Upgrade()) { - strLoadError = _("Error upgrading chainstate database").translated; + strLoadError = _("Error upgrading chainstate database"); failed_chainstate_init = true; break; } // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (!chainstate->ReplayBlocks(chainparams)) { - strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; + strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate."); failed_chainstate_init = true; break; } @@ -1632,7 +1651,7 @@ bool AppInitMain(NodeContext& node) if (!is_coinsview_empty(chainstate)) { // LoadChainTip initializes the chain based on CoinsTip()'s best block if (!chainstate->LoadChainTip(chainparams)) { - strLoadError = _("Error initializing block database").translated; + strLoadError = _("Error initializing block database"); failed_chainstate_init = true; break; // out of the per-chainstate loop } @@ -1645,14 +1664,14 @@ bool AppInitMain(NodeContext& node) } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); - strLoadError = _("Error opening block database").translated; + strLoadError = _("Error opening block database"); break; } bool failed_rewind{false}; // Can't hold cs_main while calling RewindBlockIndex, so retrieve the relevant // chainstates beforehand. - for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + for (CChainState* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { if (!fReset) { // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. // It both disconnects blocks based on the chainstate, and drops block data in @@ -1661,7 +1680,7 @@ bool AppInitMain(NodeContext& node) if (!chainstate->RewindBlockIndex(chainparams)) { strLoadError = _( "Unable to rewind the database to a pre-fork state. " - "You will need to redownload the blockchain").translated; + "You will need to redownload the blockchain"); failed_rewind = true; break; // out of the per-chainstate loop } @@ -1677,7 +1696,7 @@ bool AppInitMain(NodeContext& node) try { LOCK(cs_main); - for (CChainState* chainstate : g_chainman.GetAll()) { + for (CChainState* chainstate : chainman.GetAll()) { if (!is_coinsview_empty(chainstate)) { uiInterface.InitMessage(_("Verifying blocks...").translated); if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { @@ -1686,11 +1705,11 @@ bool AppInitMain(NodeContext& node) } const CBlockIndex* tip = chainstate->m_chain.Tip(); - RPCNotifyBlockChange(true, tip); + RPCNotifyBlockChange(tip); if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { strLoadError = _("The block database contains a block which appears to be from the future. " "This may be due to your computer's date and time being set incorrectly. " - "Only rebuild the block database if you are sure that your computer's date and time are correct").translated; + "Only rebuild the block database if you are sure that your computer's date and time are correct"); failed_verification = true; break; } @@ -1702,7 +1721,7 @@ bool AppInitMain(NodeContext& node) chainparams, &chainstate->CoinsDB(), gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { - strLoadError = _("Corrupted block database detected").translated; + strLoadError = _("Corrupted block database detected"); failed_verification = true; break; } @@ -1710,7 +1729,7 @@ bool AppInitMain(NodeContext& node) } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); - strLoadError = _("Error opening block database").translated; + strLoadError = _("Error opening block database"); failed_verification = true; break; } @@ -1725,8 +1744,8 @@ bool AppInitMain(NodeContext& node) // first suggest a reindex if (!fReset) { bool fRet = uiInterface.ThreadSafeQuestion( - strLoadError + ".\n\n" + _("Do you want to rebuild the block database now?").translated, - strLoadError + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", + strLoadError + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), + strLoadError.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { fReindex = true; @@ -1783,7 +1802,7 @@ bool AppInitMain(NodeContext& node) nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK); if (!fReindex) { LOCK(cs_main); - for (CChainState* chainstate : g_chainman.GetAll()) { + for (CChainState* chainstate : chainman.GetAll()) { uiInterface.InitMessage(_("Pruning blockstore...").translated); chainstate->PruneAndFlush(); } @@ -1799,11 +1818,11 @@ bool AppInitMain(NodeContext& node) // ********************************************************* Step 11: import blocks if (!CheckDiskSpace(GetDataDir())) { - InitError(strprintf(_("Error: Disk space is low for %s").translated, GetDataDir())); + InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir())); return false; } if (!CheckDiskSpace(GetBlocksDir())) { - InitError(strprintf(_("Error: Disk space is low for %s").translated, GetBlocksDir())); + InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir())); return false; } @@ -1811,7 +1830,7 @@ bool AppInitMain(NodeContext& node) // No locking, as this happens before any background thread is started. boost::signals2::connection block_notify_genesis_wait_connection; if (::ChainActive().Tip() == nullptr) { - block_notify_genesis_wait_connection = uiInterface.NotifyBlockTip_connect(BlockNotifyGenesisWait); + block_notify_genesis_wait_connection = uiInterface.NotifyBlockTip_connect(std::bind(BlockNotifyGenesisWait, std::placeholders::_2)); } else { fHaveGenesis = true; } @@ -1826,7 +1845,7 @@ bool AppInitMain(NodeContext& node) vImportFiles.push_back(strFile); } - threadGroup.create_thread(std::bind(&ThreadImport, vImportFiles)); + threadGroup.create_thread([=, &chainman] { ThreadImport(chainman, vImportFiles); }); // Wait for genesis block to be processed { @@ -1872,7 +1891,7 @@ bool AppInitMain(NodeContext& node) connOptions.m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, connOptions.nMaxConnections); connOptions.m_max_outbound_block_relay = std::min(MAX_BLOCKS_ONLY_CONNECTIONS, connOptions.nMaxConnections-connOptions.m_max_outbound_full_relay); connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; - connOptions.nMaxFeeler = 1; + connOptions.nMaxFeeler = MAX_FEELER_CONNECTIONS; connOptions.nBestHeight = chain_active_height; connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); @@ -1888,21 +1907,21 @@ bool AppInitMain(NodeContext& node) for (const std::string& strBind : gArgs.GetArgs("-bind")) { CService addrBind; if (!Lookup(strBind, addrBind, GetListenPort(), false)) { - return InitError(ResolveErrMsg("bind", strBind)); + return InitError(Untranslated(ResolveErrMsg("bind", strBind))); } connOptions.vBinds.push_back(addrBind); } for (const std::string& strBind : gArgs.GetArgs("-whitebind")) { NetWhitebindPermissions whitebind; std::string error; - if (!NetWhitebindPermissions::TryParse(strBind, whitebind, error)) return InitError(error); + if (!NetWhitebindPermissions::TryParse(strBind, whitebind, error)) return InitError(Untranslated(error)); connOptions.vWhiteBinds.push_back(whitebind); } for (const auto& net : gArgs.GetArgs("-whitelist")) { NetWhitelistPermissions subnet; std::string error; - if (!NetWhitelistPermissions::TryParse(net, subnet, error)) return InitError(error); + if (!NetWhitelistPermissions::TryParse(net, subnet, error)) return InitError(Untranslated(error)); connOptions.vWhitelistedRange.push_back(subnet); } diff --git a/src/init.h b/src/init.h index ef568b6f38..33fe96e8ea 100644 --- a/src/init.h +++ b/src/init.h @@ -14,6 +14,9 @@ struct NodeContext; namespace boost { class thread_group; } // namespace boost +namespace util { +class Ref; +} // namespace util /** Interrupt threads */ void Interrupt(NodeContext& node); @@ -51,7 +54,7 @@ bool AppInitLockDataDirectory(); * @note This should only be done after daemonization. Call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitLockDataDirectory should have been called. */ -bool AppInitMain(NodeContext& node); +bool AppInitMain(const util::Ref& context, NodeContext& node); /** * Register all arguments with the ArgsManager diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index 0e7641ae32..d8e459a8e8 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -53,88 +53,6 @@ bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<Rec return true; } -class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> -{ - Optional<int> getHeight() override - { - LockAssertion lock(::cs_main); - int height = ::ChainActive().Height(); - if (height >= 0) { - return height; - } - return nullopt; - } - Optional<int> getBlockHeight(const uint256& hash) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = LookupBlockIndex(hash); - if (block && ::ChainActive().Contains(block)) { - return block->nHeight; - } - return nullopt; - } - uint256 getBlockHash(int height) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; - assert(block != nullptr); - return block->GetBlockHash(); - } - bool haveBlockOnDisk(int height) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; - return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; - } - Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive().FindEarliestAtLeast(time, height); - if (block) { - if (hash) *hash = block->GetBlockHash(); - return block->nHeight; - } - return nullopt; - } - Optional<int> findFork(const uint256& hash, Optional<int>* height) override - { - LockAssertion lock(::cs_main); - const CBlockIndex* block = LookupBlockIndex(hash); - const CBlockIndex* fork = block ? ::ChainActive().FindFork(block) : nullptr; - if (height) { - if (block) { - *height = block->nHeight; - } else { - height->reset(); - } - } - if (fork) { - return fork->nHeight; - } - return nullopt; - } - CBlockLocator getTipLocator() override - { - LockAssertion lock(::cs_main); - return ::ChainActive().GetLocator(); - } - Optional<int> findLocatorFork(const CBlockLocator& locator) override - { - LockAssertion lock(::cs_main); - if (CBlockIndex* fork = FindForkInGlobalIndex(::ChainActive(), locator)) { - return fork->nHeight; - } - return nullopt; - } - bool checkFinalTx(const CTransaction& tx) override - { - LockAssertion lock(::cs_main); - return CheckFinalTx(tx); - } - - using UniqueLock::UniqueLock; -}; - class NotificationsProxy : public CValidationInterface { public: @@ -209,7 +127,7 @@ public: ::tableRPC.appendCommand(m_command.name, &m_command); } - void disconnect() override final + void disconnect() final { if (m_wrapped_command) { m_wrapped_command = nullptr; @@ -227,12 +145,64 @@ class ChainImpl : public Chain { public: explicit ChainImpl(NodeContext& node) : m_node(node) {} - std::unique_ptr<Chain::Lock> lock(bool try_lock) override + Optional<int> getHeight() override + { + LOCK(::cs_main); + int height = ::ChainActive().Height(); + if (height >= 0) { + return height; + } + return nullopt; + } + Optional<int> getBlockHeight(const uint256& hash) override + { + LOCK(::cs_main); + CBlockIndex* block = LookupBlockIndex(hash); + if (block && ::ChainActive().Contains(block)) { + return block->nHeight; + } + return nullopt; + } + uint256 getBlockHash(int height) override + { + LOCK(::cs_main); + CBlockIndex* block = ::ChainActive()[height]; + assert(block); + return block->GetBlockHash(); + } + bool haveBlockOnDisk(int height) override + { + LOCK(cs_main); + CBlockIndex* block = ::ChainActive()[height]; + return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; + } + Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) override { - auto lock = MakeUnique<LockImpl>(::cs_main, "cs_main", __FILE__, __LINE__, try_lock); - if (try_lock && lock && !*lock) return {}; - std::unique_ptr<Chain::Lock> result = std::move(lock); // Temporary to avoid CWG 1579 - return result; + LOCK(cs_main); + CBlockIndex* block = ::ChainActive().FindEarliestAtLeast(time, height); + if (block) { + if (hash) *hash = block->GetBlockHash(); + return block->nHeight; + } + return nullopt; + } + CBlockLocator getTipLocator() override + { + LOCK(cs_main); + return ::ChainActive().GetLocator(); + } + bool checkFinalTx(const CTransaction& tx) override + { + LOCK(cs_main); + return CheckFinalTx(tx); + } + Optional<int> findLocatorFork(const CBlockLocator& locator) override + { + LOCK(cs_main); + if (CBlockIndex* fork = FindForkInGlobalIndex(::ChainActive(), locator)) { + return fork->nHeight; + } + return nullopt; } bool findBlock(const uint256& hash, const FoundBlock& block) override { @@ -374,8 +344,8 @@ public: bool shutdownRequested() override { return ShutdownRequested(); } int64_t getAdjustedTime() override { return GetAdjustedTime(); } void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } - void initWarning(const std::string& message) override { InitWarning(message); } - void initError(const std::string& message) override { InitError(message); } + void initWarning(const bilingual_str& message) override { InitWarning(message); } + void initError(const bilingual_str& message) override { InitError(message); } void showProgress(const std::string& title, int progress, bool resume_possible) override { ::uiInterface.ShowProgress(title, progress, resume_possible); diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index fb77c81cdc..7dfc77db7b 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -21,6 +21,7 @@ class CScheduler; class Coin; class uint256; enum class RBFTransactionState; +struct bilingual_str; struct CBlockLocator; struct FeeCalculation; struct NodeContext; @@ -59,12 +60,7 @@ public: //! internal workings of the bitcoin node, and not being very convenient to use. //! Chain methods should be cleaned up and simplified over time. Examples: //! -//! * The Chain::lock() method, which lets clients delay chain tip updates -//! should be removed when clients are able to respond to updates -//! asynchronously -//! (https://github.com/bitcoin/bitcoin/pull/10973#issuecomment-380101269). -//! -//! * The initMessage() and showProgress() methods which the wallet uses to send +//! * The initMessages() and showProgress() methods which the wallet uses to send //! notifications to the GUI should go away when GUI and wallet can directly //! communicate with each other without going through the node //! (https://github.com/bitcoin/bitcoin/pull/15288#discussion_r253321096). @@ -72,68 +68,53 @@ public: //! * The handleRpc, registerRpcs, rpcEnableDeprecated methods and other RPC //! methods can go away if wallets listen for HTTP requests on their own //! ports instead of registering to handle requests on the node HTTP port. +//! +//! * Move fee estimation queries to an asynchronous interface and let the +//! wallet cache it, fee estimation being driven by node mempool, wallet +//! should be the consumer. +//! +//! * The `guessVerificationProgress`, `getBlockHeight`, `getBlockHash`, etc +//! methods can go away if rescan logic is moved on the node side, and wallet +//! only register rescan request. class Chain { public: virtual ~Chain() {} - //! Interface for querying locked chain state, used by legacy code that - //! assumes state won't change between calls. New code should avoid using - //! the Lock interface and instead call higher-level Chain methods - //! that return more information so the chain doesn't need to stay locked - //! between calls. - class Lock - { - public: - virtual ~Lock() {} - - //! Get current chain height, not including genesis block (returns 0 if - //! chain only contains genesis block, nullopt if chain does not contain - //! any blocks). - virtual Optional<int> getHeight() = 0; - - //! Get block height above genesis block. Returns 0 for genesis block, - //! 1 for following block, and so on. Returns nullopt for a block not - //! included in the current chain. - virtual Optional<int> getBlockHeight(const uint256& hash) = 0; - - //! Get block hash. Height must be valid or this function will abort. - virtual uint256 getBlockHash(int height) = 0; - - //! Check that the block is available on disk (i.e. has not been - //! pruned), and contains transactions. - virtual bool haveBlockOnDisk(int height) = 0; - - //! Return height of the first block in the chain with timestamp equal - //! or greater than the given time and height equal or greater than the - //! given height, or nullopt if there is no block with a high enough - //! timestamp and height. Also return the block hash as an optional output parameter - //! (to avoid the cost of a second lookup in case this information is needed.) - virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) = 0; - - //! Return height of the specified block if it is on the chain, otherwise - //! return the height of the highest block on chain that's an ancestor - //! of the specified block, or nullopt if there is no common ancestor. - //! Also return the height of the specified block as an optional output - //! parameter (to avoid the cost of a second hash lookup in case this - //! information is desired). - virtual Optional<int> findFork(const uint256& hash, Optional<int>* height) = 0; - - //! Get locator for the current chain tip. - virtual CBlockLocator getTipLocator() = 0; - - //! Return height of the highest block on chain in common with the locator, - //! which will either be the original block used to create the locator, - //! or one of its ancestors. - virtual Optional<int> findLocatorFork(const CBlockLocator& locator) = 0; - - //! Check if transaction will be final given chain height current time. - virtual bool checkFinalTx(const CTransaction& tx) = 0; - }; + //! Get current chain height, not including genesis block (returns 0 if + //! chain only contains genesis block, nullopt if chain does not contain + //! any blocks) + virtual Optional<int> getHeight() = 0; + + //! Get block height above genesis block. Returns 0 for genesis block, + //! 1 for following block, and so on. Returns nullopt for a block not + //! included in the current chain. + virtual Optional<int> getBlockHeight(const uint256& hash) = 0; + + //! Get block hash. Height must be valid or this function will abort. + virtual uint256 getBlockHash(int height) = 0; + + //! Check that the block is available on disk (i.e. has not been + //! pruned), and contains transactions. + virtual bool haveBlockOnDisk(int height) = 0; + + //! Return height of the first block in the chain with timestamp equal + //! or greater than the given time and height equal or greater than the + //! given height, or nullopt if there is no block with a high enough + //! timestamp and height. Also return the block hash as an optional output parameter + //! (to avoid the cost of a second lookup in case this information is needed.) + virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) = 0; + + //! Get locator for the current chain tip. + virtual CBlockLocator getTipLocator() = 0; + + //! Return height of the highest block on chain in common with the locator, + //! which will either be the original block used to create the locator, + //! or one of its ancestors. + virtual Optional<int> findLocatorFork(const CBlockLocator& locator) = 0; - //! Return Lock interface. Chain is locked when this is called, and - //! unlocked when the returned interface is freed. - virtual std::unique_ptr<Lock> lock(bool try_lock = false) = 0; + //! Check if transaction will be final given chain height current time. + virtual bool checkFinalTx(const CTransaction& tx) = 0; //! Return whether node has the block and optionally return block metadata //! or contents. @@ -244,10 +225,10 @@ public: virtual void initMessage(const std::string& message) = 0; //! Send init warning. - virtual void initWarning(const std::string& message) = 0; + virtual void initWarning(const bilingual_str& message) = 0; //! Send init error. - virtual void initError(const std::string& message) = 0; + virtual void initError(const bilingual_str& message) = 0; //! Send progress indicator. virtual void showProgress(const std::string& title, int progress, bool resume_possible) = 0; diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 5ebbd61584..3c94e44b53 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -27,7 +27,9 @@ #include <sync.h> #include <txmempool.h> #include <ui_interface.h> +#include <util/ref.h> #include <util/system.h> +#include <util/translation.h> #include <validation.h> #include <warnings.h> @@ -43,8 +45,8 @@ class CWallet; fs::path GetWalletDir(); std::vector<fs::path> ListWalletDir(); std::vector<std::shared_ptr<CWallet>> GetWallets(); -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::vector<std::string>& warnings); -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result); +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings); +WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result); std::unique_ptr<interfaces::Handler> HandleLoadWallet(interfaces::Node::LoadWalletFn load_wallet); namespace interfaces { @@ -54,7 +56,7 @@ namespace { class NodeImpl : public Node { public: - void initError(const std::string& message) override { InitError(message); } + void initError(const std::string& message) override { InitError(Untranslated(message)); } bool parseParameters(int argc, const char* const argv[], std::string& error) override { return gArgs.ParseParameters(argc, argv, error); @@ -79,7 +81,7 @@ public: bool appInitMain() override { m_context.chain = MakeChain(m_context); - return AppInitMain(m_context); + return AppInitMain(m_context_ref, m_context); } void appShutdown() override { @@ -224,7 +226,7 @@ public: CFeeRate getDustRelayFee() override { return ::dustRelayFee; } UniValue executeRpc(const std::string& command, const UniValue& params, const std::string& uri) override { - JSONRPCRequest req; + JSONRPCRequest req(m_context_ref); req.params = params; req.strMethod = command; req.URI = uri; @@ -259,11 +261,11 @@ public: } return wallets; } - std::unique_ptr<Wallet> loadWallet(const std::string& name, std::string& error, std::vector<std::string>& warnings) override + std::unique_ptr<Wallet> loadWallet(const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) override { return MakeWallet(LoadWallet(*m_context.chain, name, error, warnings)); } - std::unique_ptr<Wallet> createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, WalletCreationStatus& status) override + std::unique_ptr<Wallet> createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, WalletCreationStatus& status) override { std::shared_ptr<CWallet> wallet; status = CreateWallet(*m_context.chain, passphrase, wallet_creation_flags, name, error, warnings, wallet); @@ -307,21 +309,22 @@ public: } std::unique_ptr<Handler> handleNotifyBlockTip(NotifyBlockTipFn fn) override { - return MakeHandler(::uiInterface.NotifyBlockTip_connect([fn](bool initial_download, const CBlockIndex* block) { - fn(initial_download, block->nHeight, block->GetBlockTime(), + return MakeHandler(::uiInterface.NotifyBlockTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { + fn(sync_state, block->nHeight, block->GetBlockTime(), GuessVerificationProgress(Params().TxData(), block)); })); } std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) override { return MakeHandler( - ::uiInterface.NotifyHeaderTip_connect([fn](bool initial_download, const CBlockIndex* block) { - fn(initial_download, block->nHeight, block->GetBlockTime(), + ::uiInterface.NotifyHeaderTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { + fn(sync_state, block->nHeight, block->GetBlockTime(), /* verification progress is unused when a header was received */ 0); })); } NodeContext* context() override { return &m_context; } NodeContext m_context; + util::Ref m_context_ref{m_context}; }; } // namespace diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 53a20886cd..45b0e18fae 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -27,9 +27,11 @@ class Coin; class RPCTimerInterface; class UniValue; class proxyType; +enum class SynchronizationState; +enum class WalletCreationStatus; struct CNodeStateStats; struct NodeContext; -enum class WalletCreationStatus; +struct bilingual_str; namespace interfaces { class Handler; @@ -201,10 +203,10 @@ public: //! Attempts to load a wallet from file or directory. //! The loaded wallet is also notified to handlers previously registered //! with handleLoadWallet. - virtual std::unique_ptr<Wallet> loadWallet(const std::string& name, std::string& error, std::vector<std::string>& warnings) = 0; + virtual std::unique_ptr<Wallet> loadWallet(const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) = 0; //! Create a wallet from file - virtual std::unique_ptr<Wallet> createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, WalletCreationStatus& status) = 0; + virtual std::unique_ptr<Wallet> createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, WalletCreationStatus& status) = 0; //! Register handler for init messages. using InitMessageFn = std::function<void(const std::string& message)>; @@ -212,11 +214,11 @@ public: //! Register handler for message box messages. using MessageBoxFn = - std::function<bool(const std::string& message, const std::string& caption, unsigned int style)>; + std::function<bool(const bilingual_str& message, const std::string& caption, unsigned int style)>; virtual std::unique_ptr<Handler> handleMessageBox(MessageBoxFn fn) = 0; //! Register handler for question messages. - using QuestionFn = std::function<bool(const std::string& message, + using QuestionFn = std::function<bool(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style)>; @@ -248,12 +250,12 @@ public: //! Register handler for block tip messages. using NotifyBlockTipFn = - std::function<void(bool initial_download, int height, int64_t block_time, double verification_progress)>; + std::function<void(SynchronizationState, int height, int64_t block_time, double verification_progress)>; virtual std::unique_ptr<Handler> handleNotifyBlockTip(NotifyBlockTipFn fn) = 0; //! Register handler for header tip messages. using NotifyHeaderTipFn = - std::function<void(bool initial_download, int height, int64_t block_time, double verification_progress)>; + std::function<void(SynchronizationState, int height, int64_t block_time, double verification_progress)>; virtual std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) = 0; //! Return pointer to internal chain interface, useful for testing. diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 152917ac60..349dce0247 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -60,7 +60,7 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) } //! Construct wallet tx status struct. -WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx) +WalletTxStatus MakeWalletTxStatus(CWallet& wallet, const CWalletTx& wtx) { WalletTxStatus result; result.block_height = wtx.m_confirm.block_height > 0 ? wtx.m_confirm.block_height : std::numeric_limits<int>::max(); @@ -68,8 +68,8 @@ WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const C result.depth_in_main_chain = wtx.GetDepthInMainChain(); result.time_received = wtx.nTimeReceived; result.lock_time = wtx.tx->nLockTime; - result.is_final = locked_chain.checkFinalTx(*wtx.tx); - result.is_trusted = wtx.IsTrusted(locked_chain); + result.is_final = wallet.chain().checkFinalTx(*wtx.tx); + result.is_trusted = wtx.IsTrusted(); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); result.is_in_main_chain = wtx.IsInMainChain(); @@ -196,25 +196,21 @@ public: } void lockCoin(const COutPoint& output) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->LockCoin(output); } void unlockCoin(const COutPoint& output) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->UnlockCoin(output); } bool isLockedCoin(const COutPoint& output) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->IsLockedCoin(output.hash, output.n); } void listLockedCoins(std::vector<COutPoint>& outputs) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->ListLockedCoins(outputs); } @@ -223,12 +219,11 @@ public: bool sign, int& change_pos, CAmount& fee, - std::string& fail_reason) override + bilingual_str& fail_reason) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); CTransactionRef tx; - if (!m_wallet->CreateTransaction(*locked_chain, recipients, tx, fee, change_pos, + if (!m_wallet->CreateTransaction(recipients, tx, fee, change_pos, fail_reason, coin_control, sign)) { return {}; } @@ -238,14 +233,12 @@ public: WalletValueMap value_map, WalletOrderForm order_form) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); m_wallet->CommitTransaction(std::move(tx), std::move(value_map), std::move(order_form)); } bool transactionCanBeAbandoned(const uint256& txid) override { return m_wallet->TransactionCanBeAbandoned(txid); } bool abandonTransaction(const uint256& txid) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->AbandonTransaction(txid); } @@ -255,7 +248,7 @@ public: } bool createBumpTransaction(const uint256& txid, const CCoinControl& coin_control, - std::vector<std::string>& errors, + std::vector<bilingual_str>& errors, CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) override @@ -265,7 +258,7 @@ public: bool signBumpTransaction(CMutableTransaction& mtx) override { return feebumper::SignTransaction(*m_wallet.get(), mtx); } bool commitBumpTransaction(const uint256& txid, CMutableTransaction&& mtx, - std::vector<std::string>& errors, + std::vector<bilingual_str>& errors, uint256& bumped_txid) override { return feebumper::CommitTransaction(*m_wallet.get(), txid, std::move(mtx), errors, bumped_txid) == @@ -273,7 +266,6 @@ public: } CTransactionRef getTx(const uint256& txid) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); auto mi = m_wallet->mapWallet.find(txid); if (mi != m_wallet->mapWallet.end()) { @@ -283,7 +275,6 @@ public: } WalletTx getWalletTx(const uint256& txid) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); auto mi = m_wallet->mapWallet.find(txid); if (mi != m_wallet->mapWallet.end()) { @@ -293,7 +284,6 @@ public: } std::vector<WalletTx> getWalletTxs() override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); std::vector<WalletTx> result; result.reserve(m_wallet->mapWallet.size()); @@ -307,10 +297,6 @@ public: int& num_blocks, int64_t& block_time) override { - auto locked_chain = m_wallet->chain().lock(true /* try_lock */); - if (!locked_chain) { - return false; - } TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; @@ -322,7 +308,7 @@ public: num_blocks = m_wallet->GetLastBlockHeight(); block_time = -1; CHECK_NONFATAL(m_wallet->chain().findBlock(m_wallet->GetLastBlockHash(), FoundBlock().time(block_time))); - tx_status = MakeWalletTxStatus(*locked_chain, mi->second); + tx_status = MakeWalletTxStatus(*m_wallet, mi->second); return true; } WalletTx getWalletTxDetails(const uint256& txid, @@ -331,14 +317,13 @@ public: bool& in_mempool, int& num_blocks) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); auto mi = m_wallet->mapWallet.find(txid); if (mi != m_wallet->mapWallet.end()) { - num_blocks = locked_chain->getHeight().get_value_or(-1); + num_blocks = m_wallet->GetLastBlockHeight(); in_mempool = mi->second.InMempool(); order_form = mi->second.vOrderForm; - tx_status = MakeWalletTxStatus(*locked_chain, mi->second); + tx_status = MakeWalletTxStatus(*m_wallet, mi->second); return MakeWalletTx(*m_wallet, mi->second); } return {}; @@ -366,16 +351,13 @@ public: } return result; } - bool tryGetBalances(WalletBalances& balances, int& num_blocks, bool force, int cached_num_blocks) override + bool tryGetBalances(WalletBalances& balances, int& num_blocks) override { - auto locked_chain = m_wallet->chain().lock(true /* try_lock */); - if (!locked_chain) return false; TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; } num_blocks = m_wallet->GetLastBlockHeight(); - if (!force && num_blocks == cached_num_blocks) return false; balances = getBalances(); return true; } @@ -386,34 +368,29 @@ public: } isminetype txinIsMine(const CTxIn& txin) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->IsMine(txin); } isminetype txoutIsMine(const CTxOut& txout) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->IsMine(txout); } CAmount getDebit(const CTxIn& txin, isminefilter filter) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->GetDebit(txin, filter); } CAmount getCredit(const CTxOut& txout, isminefilter filter) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); return m_wallet->GetCredit(txout, filter); } CoinsList listCoins() override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); CoinsList result; - for (const auto& entry : m_wallet->ListCoins(*locked_chain)) { + for (const auto& entry : m_wallet->ListCoins()) { auto& group = result[entry.first]; for (const auto& coin : entry.second) { group.emplace_back(COutPoint(coin.tx->GetHash(), coin.i), @@ -424,7 +401,6 @@ public: } std::vector<WalletTxOut> getCoins(const std::vector<COutPoint>& outputs) override { - auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); std::vector<WalletTxOut> result; result.reserve(outputs.size()); @@ -496,6 +472,7 @@ public: { return MakeHandler(m_wallet->NotifyCanGetAddressesChanged.connect(fn)); } + CWallet* wallet() override { return m_wallet.get(); } std::shared_ptr<CWallet> m_wallet; }; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 5d870c5e3d..421d35af15 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -31,6 +31,7 @@ enum class TransactionError; enum isminetype : unsigned int; struct CRecipient; struct PartiallySignedTransaction; +struct bilingual_str; typedef uint8_t isminefilter; namespace interfaces { @@ -136,7 +137,7 @@ public: bool sign, int& change_pos, CAmount& fee, - std::string& fail_reason) = 0; + bilingual_str& fail_reason) = 0; //! Commit transaction. virtual void commitTransaction(CTransactionRef tx, @@ -155,7 +156,7 @@ public: //! Create bump transaction. virtual bool createBumpTransaction(const uint256& txid, const CCoinControl& coin_control, - std::vector<std::string>& errors, + std::vector<bilingual_str>& errors, CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) = 0; @@ -166,7 +167,7 @@ public: //! Commit bump transaction. virtual bool commitBumpTransaction(const uint256& txid, CMutableTransaction&& mtx, - std::vector<std::string>& errors, + std::vector<bilingual_str>& errors, uint256& bumped_txid) = 0; //! Get a transaction. @@ -201,11 +202,8 @@ public: //! Get balances. virtual WalletBalances getBalances() = 0; - //! Get balances if possible without waiting for chain and wallet locks. - virtual bool tryGetBalances(WalletBalances& balances, - int& num_blocks, - bool force, - int cached_num_blocks) = 0; + //! Get balances if possible without blocking. + virtual bool tryGetBalances(WalletBalances& balances, int& num_blocks) = 0; //! Get balance. virtual CAmount getBalance() = 0; @@ -300,6 +298,9 @@ public: //! Register handler for keypool changed messages. using CanGetAddressesChangedFn = std::function<void()>; virtual std::unique_ptr<Handler> handleCanGetAddressesChanged(CanGetAddressesChangedFn fn) = 0; + + //! Return pointer to internal wallet class, useful for testing. + virtual CWallet* wallet() { return nullptr; } }; //! Information about one wallet address. diff --git a/src/logging.cpp b/src/logging.cpp index eb9da06d9b..fe58ae9e73 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -22,8 +22,8 @@ BCLog::Logger& LogInstance() * access the logger. When the shutdown sequence is fully audited and tested, * explicit destruction of these objects can be implemented by changing this * from a raw pointer to a std::unique_ptr. - * Since the destructor is never called, the logger and all its members must - * have a trivial destructor. + * Since the ~Logger() destructor is never called, the Logger class and all + * its subclasses must have implicitly-defined destructors. * * This method of initialization was originally introduced in * ee3374234c60aba2cc4c5cd5cac1c0aefc2d817c. @@ -41,7 +41,7 @@ static int FileWriteStr(const std::string &str, FILE *fp) bool BCLog::Logger::StartLogging() { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); assert(m_buffering); assert(m_fileout == nullptr); @@ -80,7 +80,7 @@ bool BCLog::Logger::StartLogging() void BCLog::Logger::DisconnectTestLogger() { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); m_buffering = true; if (m_fileout != nullptr) fclose(m_fileout); m_fileout = nullptr; @@ -246,7 +246,7 @@ namespace BCLog { void BCLog::Logger::LogPrintStr(const std::string& str) { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); std::string str_prefixed = LogEscapeMessage(str); if (m_log_threadnames && m_started_new_line) { diff --git a/src/logging.h b/src/logging.h index ab07010316..7e646ef67a 100644 --- a/src/logging.h +++ b/src/logging.h @@ -8,6 +8,7 @@ #include <fs.h> #include <tinyformat.h> +#include <threadsafety.h> #include <util/string.h> #include <atomic> @@ -61,10 +62,11 @@ namespace BCLog { class Logger { private: - mutable std::mutex m_cs; // Can not use Mutex from sync.h because in debug mode it would cause a deadlock when a potential deadlock was detected - FILE* m_fileout = nullptr; // GUARDED_BY(m_cs) - std::list<std::string> m_msgs_before_open; // GUARDED_BY(m_cs) - bool m_buffering{true}; //!< Buffer messages before logging can be started. GUARDED_BY(m_cs) + mutable StdMutex m_cs; // Can not use Mutex from sync.h because in debug mode it would cause a deadlock when a potential deadlock was detected + + FILE* m_fileout GUARDED_BY(m_cs) = nullptr; + std::list<std::string> m_msgs_before_open GUARDED_BY(m_cs); + bool m_buffering GUARDED_BY(m_cs) = true; //!< Buffer messages before logging can be started. /** * m_started_new_line is a state variable that will suppress printing of @@ -79,7 +81,7 @@ namespace BCLog { std::string LogTimestampStr(const std::string& str); /** Slots that connect to the print signal */ - std::list<std::function<void(const std::string&)>> m_print_callbacks /* GUARDED_BY(m_cs) */ {}; + std::list<std::function<void(const std::string&)>> m_print_callbacks GUARDED_BY(m_cs) {}; public: bool m_print_to_console = false; @@ -98,14 +100,14 @@ namespace BCLog { /** Returns whether logs will be written to any output */ bool Enabled() const { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); return m_buffering || m_print_to_console || m_print_to_file || !m_print_callbacks.empty(); } /** Connect a slot to the print signal and return the connection */ std::list<std::function<void(const std::string&)>>::iterator PushBackCallback(std::function<void(const std::string&)> fun) { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); m_print_callbacks.push_back(std::move(fun)); return --m_print_callbacks.end(); } @@ -113,7 +115,7 @@ namespace BCLog { /** Delete a connection */ void DeleteCallback(std::list<std::function<void(const std::string&)>>::iterator it) { - std::lock_guard<std::mutex> scoped_lock(m_cs); + StdLockGuard scoped_lock(m_cs); m_print_callbacks.erase(it); } diff --git a/src/logging/timer.h b/src/logging/timer.h index 21bb3d121e..159920e397 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -93,12 +93,10 @@ private: } // namespace BCLog -#define LOG_TIME_MICROS(end_msg, ...) \ - BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) -#define LOG_TIME_MILLIS(end_msg, ...) \ - BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) -#define LOG_TIME_SECONDS(end_msg, ...) \ - BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) +#define LOG_TIME_MILLIS_WITH_CATEGORY(end_msg, log_category) \ + BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category) +#define LOG_TIME_SECONDS(end_msg) \ + BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg) #endif // BITCOIN_LOGGING_TIMER_H diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index 4ac6219886..8072b12119 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -9,6 +9,24 @@ #include <consensus/consensus.h> +std::vector<unsigned char> BitsToBytes(const std::vector<bool>& bits) +{ + std::vector<unsigned char> ret((bits.size() + 7) / 8); + for (unsigned int p = 0; p < bits.size(); p++) { + ret[p / 8] |= bits[p] << (p % 8); + } + return ret; +} + +std::vector<bool> BytesToBits(const std::vector<unsigned char>& bytes) +{ + std::vector<bool> ret(bytes.size() * 8); + for (unsigned int p = 0; p < ret.size(); p++) { + ret[p] = (bytes[p / 8] & (1 << (p % 8))) != 0; + } + return ret; +} + CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std::set<uint256>* txids) { header = block.GetBlockHeader(); diff --git a/src/merkleblock.h b/src/merkleblock.h index e641c8aa94..b2d2828784 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -13,6 +13,10 @@ #include <vector> +// Helper functions for serialization. +std::vector<unsigned char> BitsToBytes(const std::vector<bool>& bits); +std::vector<bool> BytesToBits(const std::vector<unsigned char>& bytes); + /** Data structure that represents a partial merkle tree. * * It represents a subset of the txid's of a known block, in a way that @@ -81,27 +85,14 @@ protected: public: - /** serialization implementation */ - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nTransactions); - READWRITE(vHash); - std::vector<unsigned char> vBytes; - if (ser_action.ForRead()) { - READWRITE(vBytes); - CPartialMerkleTree &us = *(const_cast<CPartialMerkleTree*>(this)); - us.vBits.resize(vBytes.size() * 8); - for (unsigned int p = 0; p < us.vBits.size(); p++) - us.vBits[p] = (vBytes[p / 8] & (1 << (p % 8))) != 0; - us.fBad = false; - } else { - vBytes.resize((vBits.size()+7)/8); - for (unsigned int p = 0; p < vBits.size(); p++) - vBytes[p / 8] |= vBits[p] << (p % 8); - READWRITE(vBytes); - } + SERIALIZE_METHODS(CPartialMerkleTree, obj) + { + READWRITE(obj.nTransactions, obj.vHash); + std::vector<unsigned char> bytes; + SER_WRITE(obj, bytes = BitsToBytes(obj.vBits)); + READWRITE(bytes); + SER_READ(obj, obj.vBits = BytesToBits(bytes)); + SER_READ(obj, obj.fBad = false); } /** Construct a partial merkle tree from a list of transaction ids, and a mask that selects a subset of them */ @@ -157,13 +148,7 @@ public: CMerkleBlock() {} - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(header); - READWRITE(txn); - } + SERIALIZE_METHODS(CMerkleBlock, obj) { READWRITE(obj.header, obj.txn); } private: // Combined constructor to consolidate code diff --git a/src/net.cpp b/src/net.cpp index dcc613ba88..c9cfb67ec8 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -16,6 +16,7 @@ #include <crypto/sha256.h> #include <netbase.h> #include <net_permissions.h> +#include <protocol.h> #include <random.h> #include <scheduler.h> #include <ui_interface.h> @@ -631,14 +632,14 @@ int CNode::GetSendVersion() const int V1TransportDeserializer::readHeader(const char *pch, unsigned int nBytes) { // copy data to temporary parsing buffer - unsigned int nRemaining = 24 - nHdrPos; + unsigned int nRemaining = CMessageHeader::HEADER_SIZE - nHdrPos; unsigned int nCopy = std::min(nRemaining, nBytes); memcpy(&hdrbuf[nHdrPos], pch, nCopy); nHdrPos += nCopy; // if header incomplete, exit - if (nHdrPos < 24) + if (nHdrPos < CMessageHeader::HEADER_SIZE) return nCopy; // deserialize to CMessageHeader @@ -1454,7 +1455,7 @@ void CConnman::ThreadSocketHandler() void CConnman::WakeMessageHandler() { { - std::lock_guard<std::mutex> lock(mutexMsgProc); + LOCK(mutexMsgProc); fMsgProcWake = true; } condMsgProc.notify_one(); @@ -2057,7 +2058,7 @@ void CConnman::ThreadMessageHandler() WAIT_LOCK(mutexMsgProc, lock); if (!fMoreWork) { - condMsgProc.wait_until(lock, std::chrono::steady_clock::now() + std::chrono::milliseconds(100), [this] { return fMsgProcWake; }); + condMsgProc.wait_until(lock, std::chrono::steady_clock::now() + std::chrono::milliseconds(100), [this]() EXCLUSIVE_LOCKS_REQUIRED(mutexMsgProc) { return fMsgProcWake; }); } fMsgProcWake = false; } @@ -2068,9 +2069,8 @@ void CConnman::ThreadMessageHandler() -bool CConnman::BindListenPort(const CService& addrBind, std::string& strError, NetPermissionFlags permissions) +bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, NetPermissionFlags permissions) { - strError = ""; int nOne = 1; // Create socket for listening for incoming connections @@ -2078,16 +2078,16 @@ bool CConnman::BindListenPort(const CService& addrBind, std::string& strError, N socklen_t len = sizeof(sockaddr); if (!addrBind.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { - strError = strprintf("Error: Bind address family for %s not supported", addrBind.ToString()); - LogPrintf("%s\n", strError); + strError = strprintf(Untranslated("Error: Bind address family for %s not supported"), addrBind.ToString()); + LogPrintf("%s\n", strError.original); return false; } SOCKET hListenSocket = CreateSocket(addrBind); if (hListenSocket == INVALID_SOCKET) { - strError = strprintf("Error: Couldn't open socket for incoming connections (socket returned error %s)", NetworkErrorString(WSAGetLastError())); - LogPrintf("%s\n", strError); + strError = strprintf(Untranslated("Error: Couldn't open socket for incoming connections (socket returned error %s)"), NetworkErrorString(WSAGetLastError())); + LogPrintf("%s\n", strError.original); return false; } @@ -2111,10 +2111,10 @@ bool CConnman::BindListenPort(const CService& addrBind, std::string& strError, N { int nErr = WSAGetLastError(); if (nErr == WSAEADDRINUSE) - strError = strprintf(_("Unable to bind to %s on this computer. %s is probably already running.").translated, addrBind.ToString(), PACKAGE_NAME); + strError = strprintf(_("Unable to bind to %s on this computer. %s is probably already running."), addrBind.ToString(), PACKAGE_NAME); else - strError = strprintf(_("Unable to bind to %s on this computer (bind returned error %s)").translated, addrBind.ToString(), NetworkErrorString(nErr)); - LogPrintf("%s\n", strError); + strError = strprintf(_("Unable to bind to %s on this computer (bind returned error %s)"), addrBind.ToString(), NetworkErrorString(nErr)); + LogPrintf("%s\n", strError.original); CloseSocket(hListenSocket); return false; } @@ -2123,8 +2123,8 @@ bool CConnman::BindListenPort(const CService& addrBind, std::string& strError, N // Listen for incoming connections if (listen(hListenSocket, SOMAXCONN) == SOCKET_ERROR) { - strError = strprintf(_("Error: Listening for incoming connections failed (listen returned error %s)").translated, NetworkErrorString(WSAGetLastError())); - LogPrintf("%s\n", strError); + strError = strprintf(_("Error: Listening for incoming connections failed (listen returned error %s)"), NetworkErrorString(WSAGetLastError())); + LogPrintf("%s\n", strError.original); CloseSocket(hListenSocket); return false; } @@ -2218,7 +2218,7 @@ NodeId CConnman::GetNewNodeId() bool CConnman::Bind(const CService &addr, unsigned int flags, NetPermissionFlags permissions) { if (!(flags & BF_EXPLICIT) && !IsReachable(addr)) return false; - std::string strError; + bilingual_str strError; if (!BindListenPort(addr, strError, permissions)) { if ((flags & BF_REPORT_ERROR) && clientInterface) { clientInterface->ThreadSafeMessageBox(strError, "", CClientUIInterface::MSG_ERROR); @@ -2265,7 +2265,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) if (fListen && !InitBinds(connOptions.vBinds, connOptions.vWhiteBinds)) { if (clientInterface) { clientInterface->ThreadSafeMessageBox( - _("Failed to listen on any port. Use -listen=0 if you want this.").translated, + _("Failed to listen on any port. Use -listen=0 if you want this."), "", CClientUIInterface::MSG_ERROR); } return false; @@ -2331,7 +2331,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) if (connOptions.m_use_addrman_outgoing && !connOptions.m_specified_outgoing.empty()) { if (clientInterface) { clientInterface->ThreadSafeMessageBox( - _("Cannot provide specific connections and have addrman find outgoing connections at the same.").translated, + _("Cannot provide specific connections and have addrman find outgoing connections at the same."), "", CClientUIInterface::MSG_ERROR); } return false; @@ -2366,7 +2366,7 @@ static CNetCleanup instance_of_cnetcleanup; void CConnman::Interrupt() { { - std::lock_guard<std::mutex> lock(mutexMsgProc); + LOCK(mutexMsgProc); flagInterruptMsgProc = true; } condMsgProc.notify_all(); @@ -21,8 +21,8 @@ #include <random.h> #include <streams.h> #include <sync.h> -#include <uint256.h> #include <threadinterrupt.h> +#include <uint256.h> #include <atomic> #include <deque> @@ -39,6 +39,7 @@ class CScheduler; class CNode; class BanMan; +struct bilingual_str; /** Default for -whitelistrelay. */ static const bool DEFAULT_WHITELISTRELAY = true; @@ -61,6 +62,8 @@ static const int MAX_OUTBOUND_FULL_RELAY_CONNECTIONS = 8; static const int MAX_ADDNODE_CONNECTIONS = 8; /** Maximum number of block-relay-only outgoing connections */ static const int MAX_BLOCKS_ONLY_CONNECTIONS = 2; +/** Maximum number of feeler connections */ +static const int MAX_FEELER_CONNECTIONS = 1; /** -listen default */ static const bool DEFAULT_LISTEN = true; /** -upnp default */ @@ -334,7 +337,7 @@ private: NetPermissionFlags m_permissions; }; - bool BindListenPort(const CService& bindAddr, std::string& strError, NetPermissionFlags permissions); + bool BindListenPort(const CService& bindAddr, bilingual_str& strError, NetPermissionFlags permissions); bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); bool InitBinds(const std::vector<CService>& binds, const std::vector<NetWhitebindPermissions>& whiteBinds); void ThreadOpenAddedConnections(); @@ -451,7 +454,7 @@ private: const uint64_t nSeed0, nSeed1; /** flag for waking the message processor. */ - bool fMsgProcWake; + bool fMsgProcWake GUARDED_BY(mutexMsgProc); std::condition_variable condMsgProc; Mutex mutexMsgProc; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 50a8a8a882..159036a237 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -8,9 +8,11 @@ #include <addrman.h> #include <banman.h> #include <blockencodings.h> +#include <blockfilter.h> #include <chainparams.h> #include <consensus/validation.h> #include <hash.h> +#include <index/blockfilterindex.h> #include <validation.h> #include <merkleblock.h> #include <netmessagemaker.h> @@ -127,6 +129,8 @@ static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_ static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; /** Maximum feefilter broadcast delay after significant change. */ static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; +/** Maximum number of cf hashes that may be requested with one getcfheaders. See BIP 157. */ +static constexpr uint32_t MAX_GETCFHEADERS_SIZE = 2000; struct COrphanTx { // When modifying, adapt the copy of this definition in tests/DoS_tests. @@ -815,7 +819,12 @@ void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const std::set<uint256> unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); for (const uint256& txid : unbroadcast_txids) { - RelayTransaction(txid, *connman); + // Sanity check: all unbroadcast txns should exist in the mempool + if (m_mempool.exists(txid)) { + RelayTransaction(txid, *connman); + } else { + m_mempool.RemoveUnbroadcastTx(txid, true); + } } // schedule next run for 10-15 minutes in the future @@ -1146,9 +1155,10 @@ static bool BlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Para (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) < STALE_RELAY_AGE_LIMIT); } -PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CScheduler& scheduler, CTxMemPool& pool) +PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool) : connman(connmanIn), m_banman(banman), + m_chainman(chainman), m_mempool(pool), m_stale_tip_check_time(0) { @@ -1604,6 +1614,37 @@ void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, c } } +//! Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). +CTransactionRef static FindTxForGetData(CNode* peer, const uint256& txid, const std::chrono::seconds mempool_req, const std::chrono::seconds longlived_mempool_time) LOCKS_EXCLUDED(cs_main) +{ + // Check if the requested transaction is so recent that we're just + // about to announce it to the peer; if so, they certainly shouldn't + // know we already have it. + { + LOCK(peer->m_tx_relay->cs_tx_inventory); + if (peer->m_tx_relay->setInventoryTxToSend.count(txid)) return {}; + } + + { + LOCK(cs_main); + // Look up transaction in relay pool + auto mi = mapRelay.find(txid); + if (mi != mapRelay.end()) return mi->second; + } + + auto txinfo = mempool.info(txid); + if (txinfo.tx) { + // To protect privacy, do not answer getdata using the mempool when + // that TX couldn't have been INVed in reply to a MEMPOOL request, + // or when it's too recent to have expired from mapRelay. + if ((mempool_req.count() && txinfo.m_time <= mempool_req) || txinfo.m_time <= longlived_mempool_time) { + return txinfo.tx; + } + } + + return {}; +} + void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnman* connman, CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) { AssertLockNotHeld(cs_main); @@ -1612,72 +1653,49 @@ void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnm std::vector<CInv> vNotFound; const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - // Note that if we receive a getdata for a MSG_TX or MSG_WITNESS_TX from a - // block-relay-only outbound peer, we will stop processing further getdata - // messages from this peer (likely resulting in our peer eventually - // disconnecting us). - if (pfrom->m_tx_relay != nullptr) { - // mempool entries added before this time have likely expired from mapRelay - const std::chrono::seconds longlived_mempool_time = GetTime<std::chrono::seconds>() - RELAY_TX_CACHE_TIME; - const std::chrono::seconds mempool_req = pfrom->m_tx_relay->m_last_mempool_req.load(); + // mempool entries added before this time have likely expired from mapRelay + const std::chrono::seconds longlived_mempool_time = GetTime<std::chrono::seconds>() - RELAY_TX_CACHE_TIME; + // Get last mempool request time + const std::chrono::seconds mempool_req = pfrom->m_tx_relay != nullptr ? pfrom->m_tx_relay->m_last_mempool_req.load() + : std::chrono::seconds::min(); - LOCK(cs_main); + // Process as many TX items from the front of the getdata queue as + // possible, since they're common and it's efficient to batch process + // them. + while (it != pfrom->vRecvGetData.end() && (it->type == MSG_TX || it->type == MSG_WITNESS_TX)) { + if (interruptMsgProc) return; + // The send buffer provides backpressure. If there's no space in + // the buffer, pause processing until the next call. + if (pfrom->fPauseSend) break; - while (it != pfrom->vRecvGetData.end() && (it->type == MSG_TX || it->type == MSG_WITNESS_TX)) { - if (interruptMsgProc) - return; - // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - break; + const CInv &inv = *it++; - const CInv &inv = *it; - it++; + if (pfrom->m_tx_relay == nullptr) { + // Ignore GETDATA requests for transactions from blocks-only peers. + continue; + } - // Send stream from relay memory - bool push = false; - auto mi = mapRelay.find(inv.hash); + CTransactionRef tx = FindTxForGetData(pfrom, inv.hash, mempool_req, longlived_mempool_time); + if (tx) { int nSendFlags = (inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); - if (mi != mapRelay.end()) { - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *mi->second)); - push = true; - } else { - auto txinfo = mempool.info(inv.hash); - // To protect privacy, do not answer getdata using the mempool when - // that TX couldn't have been INVed in reply to a MEMPOOL request, - // or when it's too recent to have expired from mapRelay. - if (txinfo.tx && ( - (mempool_req.count() && txinfo.m_time <= mempool_req) - || (txinfo.m_time <= longlived_mempool_time))) - { - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *txinfo.tx)); - push = true; - } - } - - if (push) { - // We interpret fulfilling a GETDATA for a transaction as a - // successful initial broadcast and remove it from our - // unbroadcast set. - mempool.RemoveUnbroadcastTx(inv.hash); - } else { - vNotFound.push_back(inv); - } + connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); + mempool.RemoveUnbroadcastTx(inv.hash); + } else { + vNotFound.push_back(inv); } - } // release cs_main + } + // Only process one BLOCK item per call, since they're uncommon and can be + // expensive to process. if (it != pfrom->vRecvGetData.end() && !pfrom->fPauseSend) { - const CInv &inv = *it; + const CInv &inv = *it++; if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) { - it++; ProcessGetBlockData(pfrom, chainparams, inv, connman); } + // else: If the first item on the queue is an unknown type, we erase it + // and continue processing the queue on the next call. } - // Unknown types in the GetData stay in vRecvGetData and block any future - // message from this peer, see vRecvGetData check in ProcessMessages(). - // Depending on future p2p changes, we might either drop unknown getdata on - // the floor or disconnect the peer. - pfrom->vRecvGetData.erase(pfrom->vRecvGetData.begin(), it); if (!vNotFound.empty()) { @@ -1723,7 +1741,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } -bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& mempool, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool via_compact_block) +bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, ChainstateManager& chainman, CTxMemPool& mempool, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool via_compact_block) { const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); size_t nCount = headers.size(); @@ -1783,7 +1801,7 @@ bool static ProcessHeadersMessage(CNode* pfrom, CConnman* connman, CTxMemPool& m } BlockValidationState state; - if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { + if (!chainman.ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { if (state.IsInvalid()) { MaybePunishNodeForBlock(pfrom->GetId(), state, via_compact_block, "invalid header received"); return false; @@ -1965,7 +1983,181 @@ void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uin } } -bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) +/** + * Validation logic for compact filters request handling. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] pfrom The peer that we received the request from + * @param[in] chain_params Chain parameters + * @param[in] filter_type The filter type the request is for. Must be basic filters. + * @param[in] start_height The start height for the request + * @param[in] stop_hash The stop_hash for the request + * @param[in] max_height_diff The maximum number of items permitted to request, as specified in BIP 157 + * @param[out] stop_index The CBlockIndex for the stop_hash block, if the request can be serviced. + * @param[out] filter_index The filter index, if the request can be serviced. + * @return True if the request can be serviced. + */ +static bool PrepareBlockFilterRequest(CNode* pfrom, const CChainParams& chain_params, + BlockFilterType filter_type, uint32_t start_height, + const uint256& stop_hash, uint32_t max_height_diff, + const CBlockIndex*& stop_index, + BlockFilterIndex*& filter_index) +{ + const bool supported_filter_type = + (filter_type == BlockFilterType::BASIC && + gArgs.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS)); + if (!supported_filter_type) { + LogPrint(BCLog::NET, "peer %d requested unsupported block filter type: %d\n", + pfrom->GetId(), static_cast<uint8_t>(filter_type)); + pfrom->fDisconnect = true; + return false; + } + + { + LOCK(cs_main); + stop_index = LookupBlockIndex(stop_hash); + + // Check that the stop block exists and the peer would be allowed to fetch it. + if (!stop_index || !BlockRequestAllowed(stop_index, chain_params.GetConsensus())) { + LogPrint(BCLog::NET, "peer %d requested invalid block hash: %s\n", + pfrom->GetId(), stop_hash.ToString()); + pfrom->fDisconnect = true; + return false; + } + } + + uint32_t stop_height = stop_index->nHeight; + if (start_height > stop_height) { + LogPrint(BCLog::NET, "peer %d sent invalid getcfilters/getcfheaders with " /* Continued */ + "start height %d and stop height %d\n", + pfrom->GetId(), start_height, stop_height); + pfrom->fDisconnect = true; + return false; + } + if (stop_height - start_height >= max_height_diff) { + LogPrint(BCLog::NET, "peer %d requested too many cfilters/cfheaders: %d / %d\n", + pfrom->GetId(), stop_height - start_height + 1, max_height_diff); + pfrom->fDisconnect = true; + return false; + } + + filter_index = GetBlockFilterIndex(filter_type); + if (!filter_index) { + LogPrint(BCLog::NET, "Filter index for supported type %s not found\n", BlockFilterTypeName(filter_type)); + return false; + } + + return true; +} + +/** + * Handle a cfheaders request. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] pfrom The peer that we received the request from + * @param[in] vRecv The raw message received + * @param[in] chain_params Chain parameters + * @param[in] connman Pointer to the connection manager + */ +static void ProcessGetCFHeaders(CNode* pfrom, CDataStream& vRecv, const CChainParams& chain_params, + CConnman* connman) +{ + uint8_t filter_type_ser; + uint32_t start_height; + uint256 stop_hash; + + vRecv >> filter_type_ser >> start_height >> stop_hash; + + const BlockFilterType filter_type = static_cast<BlockFilterType>(filter_type_ser); + + const CBlockIndex* stop_index; + BlockFilterIndex* filter_index; + if (!PrepareBlockFilterRequest(pfrom, chain_params, filter_type, start_height, stop_hash, + MAX_GETCFHEADERS_SIZE, stop_index, filter_index)) { + return; + } + + uint256 prev_header; + if (start_height > 0) { + const CBlockIndex* const prev_block = + stop_index->GetAncestor(static_cast<int>(start_height - 1)); + if (!filter_index->LookupFilterHeader(prev_block, prev_header)) { + LogPrint(BCLog::NET, "Failed to find block filter header in index: filter_type=%s, block_hash=%s\n", + BlockFilterTypeName(filter_type), prev_block->GetBlockHash().ToString()); + return; + } + } + + std::vector<uint256> filter_hashes; + if (!filter_index->LookupFilterHashRange(start_height, stop_index, filter_hashes)) { + LogPrint(BCLog::NET, "Failed to find block filter hashes in index: filter_type=%s, start_height=%d, stop_hash=%s\n", + BlockFilterTypeName(filter_type), start_height, stop_hash.ToString()); + return; + } + + CSerializedNetMsg msg = CNetMsgMaker(pfrom->GetSendVersion()) + .Make(NetMsgType::CFHEADERS, + filter_type_ser, + stop_index->GetBlockHash(), + prev_header, + filter_hashes); + connman->PushMessage(pfrom, std::move(msg)); +} + +/** + * Handle a getcfcheckpt request. + * + * May disconnect from the peer in the case of a bad request. + * + * @param[in] pfrom The peer that we received the request from + * @param[in] vRecv The raw message received + * @param[in] chain_params Chain parameters + * @param[in] connman Pointer to the connection manager + */ +static void ProcessGetCFCheckPt(CNode* pfrom, CDataStream& vRecv, const CChainParams& chain_params, + CConnman* connman) +{ + uint8_t filter_type_ser; + uint256 stop_hash; + + vRecv >> filter_type_ser >> stop_hash; + + const BlockFilterType filter_type = static_cast<BlockFilterType>(filter_type_ser); + + const CBlockIndex* stop_index; + BlockFilterIndex* filter_index; + if (!PrepareBlockFilterRequest(pfrom, chain_params, filter_type, /*start_height=*/0, stop_hash, + /*max_height_diff=*/std::numeric_limits<uint32_t>::max(), + stop_index, filter_index)) { + return; + } + + std::vector<uint256> headers(stop_index->nHeight / CFCHECKPT_INTERVAL); + + // Populate headers. + const CBlockIndex* block_index = stop_index; + for (int i = headers.size() - 1; i >= 0; i--) { + int height = (i + 1) * CFCHECKPT_INTERVAL; + block_index = block_index->GetAncestor(height); + + if (!filter_index->LookupFilterHeader(block_index, headers[i])) { + LogPrint(BCLog::NET, "Failed to find block filter header in index: filter_type=%s, block_hash=%s\n", + BlockFilterTypeName(filter_type), block_index->GetBlockHash().ToString()); + return; + } + } + + CSerializedNetMsg msg = CNetMsgMaker(pfrom->GetSendVersion()) + .Make(NetMsgType::CFCHECKPT, + filter_type_ser, + stop_index->GetBlockHash(), + headers); + connman->PushMessage(pfrom, std::move(msg)); +} + +bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, ChainstateManager& chainman, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom->GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) @@ -2311,6 +2503,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec uint32_t nFetchFlags = GetFetchFlags(pfrom); const auto current_time = GetTime<std::chrono::microseconds>(); + uint256* best_block{nullptr}; for (CInv &inv : vInv) { @@ -2327,17 +2520,14 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (inv.type == MSG_BLOCK) { UpdateBlockAvailability(pfrom->GetId(), inv.hash); if (!fAlreadyHave && !fImporting && !fReindex && !mapBlocksInFlight.count(inv.hash)) { - // We used to request the full block here, but since headers-announcements are now the - // primary method of announcement on the network, and since, in the case that a node - // fell back to inv we probably have a reorg which we should get the headers for first, - // we now only provide a getheaders response here. When we receive the headers, we will - // then ask for the blocks we need. - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), inv.hash)); - LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->GetId()); + // Headers-first is the primary method of announcement on + // the network. If a node fell back to sending blocks by inv, + // it's probably for a re-org. The final block hash + // provided should be the highest, so send a getheaders and + // then fetch the blocks we need to catch up. + best_block = &inv.hash; } - } - else - { + } else { pfrom->AddInventoryKnown(inv); if (fBlocksOnly) { LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom->GetId()); @@ -2348,6 +2538,12 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec } } } + + if (best_block != nullptr) { + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), *best_block)); + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, best_block->ToString(), pfrom->GetId()); + } + return true; } @@ -2558,8 +2754,8 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec if (msg_type == NetMsgType::TX) { // Stop processing the transaction early if - // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off - // or if this peer is supposed to be a block-relay-only peer + // 1) We are in blocks only mode and peer has no relay permission + // 2) This peer is a block-relay-only peer if ((!g_relay_txes && !pfrom->HasPermission(PF_RELAY)) || (pfrom->m_tx_relay == nullptr)) { LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom->GetId()); @@ -2728,7 +2924,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec const CBlockIndex *pindex = nullptr; BlockValidationState state; - if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { + if (!chainman.ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { if (state.IsInvalid()) { MaybePunishNodeForBlock(pfrom->GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); return true; @@ -2872,7 +3068,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec } // cs_main if (fProcessBLOCKTXN) - return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, mempool, connman, banman, interruptMsgProc); + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, chainman, mempool, connman, banman, interruptMsgProc); if (fRevertToHeaderProcessing) { // Headers received from HB compact block peers are permitted to be @@ -2880,7 +3076,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // the peer if the header turns out to be for an invalid block. // Note that if a peer tries to build on an invalid chain, that // will be detected and the peer will be banned. - return ProcessHeadersMessage(pfrom, connman, mempool, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); + return ProcessHeadersMessage(pfrom, connman, chainman, mempool, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); } if (fBlockReconstructed) { @@ -2900,7 +3096,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // we have a chain with at least nMinimumChainWork), and we ignore // compact blocks with less work than our tip, it is safe to treat // reconstructed compact blocks as having been requested. - ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); + chainman.ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { @@ -2990,7 +3186,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec // disk-space attacks), but this should be safe due to the // protections in the compact block handler -- see related comment // in compact block optimistic reconstruction handling. - ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); + chainman.ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { @@ -3024,7 +3220,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } - return ProcessHeadersMessage(pfrom, connman, mempool, headers, chainparams, /*via_compact_block=*/false); + return ProcessHeadersMessage(pfrom, connman, chainman, mempool, headers, chainparams, /*via_compact_block=*/false); } if (msg_type == NetMsgType::BLOCK) @@ -3053,7 +3249,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); } bool fNewBlock = false; - ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); + chainman.ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { @@ -3216,7 +3412,6 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec { LOCK(pfrom->m_tx_relay->cs_filter); pfrom->m_tx_relay->pfilter.reset(new CBloomFilter(filter)); - pfrom->m_tx_relay->pfilter->UpdateEmptyFull(); pfrom->m_tx_relay->fRelayTxes = true; } return true; @@ -3271,6 +3466,16 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec return true; } + if (msg_type == NetMsgType::GETCFHEADERS) { + ProcessGetCFHeaders(pfrom, vRecv, chainparams, connman); + return true; + } + + if (msg_type == NetMsgType::GETCFCHECKPT) { + ProcessGetCFCheckPt(pfrom, vRecv, chainparams, connman); + return true; + } + if (msg_type == NetMsgType::NOTFOUND) { // Remove the NOTFOUND transactions from the peer LOCK(cs_main); @@ -3410,7 +3615,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fRet = false; try { - fRet = ProcessMessage(pfrom, msg_type, vRecv, msg.m_time, chainparams, m_mempool, connman, m_banman, interruptMsgProc); + fRet = ProcessMessage(pfrom, msg_type, vRecv, msg.m_time, chainparams, m_chainman, m_mempool, connman, m_banman, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) diff --git a/src/net_processing.h b/src/net_processing.h index a85d5e7c70..ec758c7537 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -12,6 +12,7 @@ #include <validationinterface.h> class CTxMemPool; +class ChainstateManager; extern RecursiveMutex cs_main; extern RecursiveMutex g_cs_orphans; @@ -21,17 +22,19 @@ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; /** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; static const bool DEFAULT_PEERBLOOMFILTERS = false; +static const bool DEFAULT_PEERBLOCKFILTERS = false; class PeerLogicValidation final : public CValidationInterface, public NetEventsInterface { private: CConnman* const connman; BanMan* const m_banman; + ChainstateManager& m_chainman; CTxMemPool& m_mempool; bool CheckIfBanned(CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); public: - PeerLogicValidation(CConnman* connman, BanMan* banman, CScheduler& scheduler, CTxMemPool& pool); + PeerLogicValidation(CConnman* connman, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool); /** * Overridden from CValidationInterface. diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 6adb171abd..f79425a52e 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -894,3 +894,8 @@ bool operator<(const CSubNet& a, const CSubNet& b) { return (a.network < b.network || (a.network == b.network && memcmp(a.netmask, b.netmask, 16) < 0)); } + +bool SanityCheckASMap(const std::vector<bool>& asmap) +{ + return SanityCheckASMap(asmap, 128); // For IP address lookups, the input is 128 bits +} diff --git a/src/netaddress.h b/src/netaddress.h index d0ab770379..e640c07d32 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -99,12 +99,7 @@ class CNetAddr friend bool operator!=(const CNetAddr& a, const CNetAddr& b) { return !(a == b); } friend bool operator<(const CNetAddr& a, const CNetAddr& b); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(ip); - } + SERIALIZE_METHODS(CNetAddr, obj) { READWRITE(obj.ip); } friend class CSubNet; }; @@ -136,14 +131,7 @@ class CSubNet friend bool operator!=(const CSubNet& a, const CSubNet& b) { return !(a == b); } friend bool operator<(const CSubNet& a, const CSubNet& b); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(network); - READWRITE(netmask); - READWRITE(valid); - } + SERIALIZE_METHODS(CSubNet, obj) { READWRITE(obj.network, obj.netmask, obj.valid); } }; /** A combination of a network address (CNetAddr) and a (TCP) port */ @@ -171,13 +159,9 @@ class CService : public CNetAddr CService(const struct in6_addr& ipv6Addr, unsigned short port); explicit CService(const struct sockaddr_in6& addr); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(ip); - READWRITE(WrapBigEndian(port)); - } + SERIALIZE_METHODS(CService, obj) { READWRITE(obj.ip, Using<BigEndianFormatter<2>>(obj.port)); } }; +bool SanityCheckASMap(const std::vector<bool>& asmap); + #endif // BITCOIN_NETADDRESS_H diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index ec52a08ace..e3c4c828b6 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -33,7 +33,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, } //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void()>& interruption_point) { stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); @@ -49,6 +49,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { + interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { diff --git a/src/node/coinstats.h b/src/node/coinstats.h index a19af0fd1b..d9cdaa3036 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -10,6 +10,7 @@ #include <uint256.h> #include <cstdint> +#include <functional> class CCoinsView; @@ -29,6 +30,6 @@ struct CCoinsStats }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats); +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, const std::function<void()>& interruption_point = {}); #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/node/context.h b/src/node/context.h index 566ff170be..c45d9e6689 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_NODE_CONTEXT_H #define BITCOIN_NODE_CONTEXT_H +#include <cassert> #include <memory> #include <vector> @@ -13,6 +14,7 @@ class BanMan; class CConnman; class CScheduler; class CTxMemPool; +class ChainstateManager; class PeerLogicValidation; namespace interfaces { class Chain; @@ -33,6 +35,7 @@ struct NodeContext { std::unique_ptr<CConnman> connman; CTxMemPool* mempool{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<PeerLogicValidation> peer_logic; + ChainstateManager* chainman{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<BanMan> banman; ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<interfaces::Chain> chain; @@ -46,4 +49,10 @@ struct NodeContext { ~NodeContext(); }; +inline ChainstateManager& EnsureChainman(const NodeContext& node) +{ + assert(node.chainman); + return *node.chainman; +} + #endif // BITCOIN_NODE_CONTEXT_H diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 702a0cbe53..c8b4d60fd0 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -35,16 +35,7 @@ public: m_coins_count(coins_count), m_nchaintx(nchaintx) { } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(m_base_blockhash); - READWRITE(m_coins_count); - READWRITE(m_nchaintx); - } - + SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count, obj.m_nchaintx); } }; #endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/noui.cpp b/src/noui.cpp index 11cfe7f94d..ddb3a50ff7 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -5,8 +5,9 @@ #include <noui.h> +#include <logging.h> #include <ui_interface.h> -#include <util/system.h> +#include <util/translation.h> #include <string> @@ -18,7 +19,7 @@ boost::signals2::connection noui_ThreadSafeMessageBoxConn; boost::signals2::connection noui_ThreadSafeQuestionConn; boost::signals2::connection noui_InitMessageConn; -bool noui_ThreadSafeMessageBox(const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { bool fSecure = style & CClientUIInterface::SECURE; style &= ~CClientUIInterface::SECURE; @@ -43,15 +44,15 @@ bool noui_ThreadSafeMessageBox(const std::string& message, const std::string& ca } if (!fSecure) { - LogPrintf("%s%s\n", strCaption, message); + LogPrintf("%s%s\n", strCaption, message.original); } - tfm::format(std::cerr, "%s%s\n", strCaption, message); + tfm::format(std::cerr, "%s%s\n", strCaption, message.original); return false; } -bool noui_ThreadSafeQuestion(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeQuestion(const bilingual_str& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) { - return noui_ThreadSafeMessageBox(message, caption, style); + return noui_ThreadSafeMessageBox(Untranslated(message), caption, style); } void noui_InitMessage(const std::string& message) @@ -66,13 +67,13 @@ void noui_connect() noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessage); } -bool noui_ThreadSafeMessageBoxRedirect(const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeMessageBoxRedirect(const bilingual_str& message, const std::string& caption, unsigned int style) { - LogPrintf("%s: %s\n", caption, message); + LogPrintf("%s: %s\n", caption, message.original); return false; } -bool noui_ThreadSafeQuestionRedirect(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeQuestionRedirect(const bilingual_str& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) { LogPrintf("%s: %s\n", caption, message); return false; diff --git a/src/noui.h b/src/noui.h index 5e5767b453..8ec5708328 100644 --- a/src/noui.h +++ b/src/noui.h @@ -7,10 +7,12 @@ #include <string> +struct bilingual_str; + /** Non-GUI handler, which logs and prints messages. */ -bool noui_ThreadSafeMessageBox(const std::string& message, const std::string& caption, unsigned int style); +bool noui_ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style); /** Non-GUI handler, which logs and prints questions. */ -bool noui_ThreadSafeQuestion(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style); +bool noui_ThreadSafeQuestion(const bilingual_str& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style); /** Non-GUI handler, which only logs a message. */ void noui_InitMessage(const std::string& message); diff --git a/src/policy/feerate.h b/src/policy/feerate.h index c040867965..61fa80c130 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -48,12 +48,7 @@ public: CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } std::string ToString() const; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nSatoshisPerK); - } + SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; #endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/prevector.h b/src/prevector.h index 9f2f7ba2de..aa20efaaa7 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -152,7 +152,7 @@ private: struct { char* indirect; size_type capacity; - }; + } indirect_contents; }; #pragma pack(pop) alignas(char*) direct_or_indirect _union = {}; @@ -163,8 +163,8 @@ private: T* direct_ptr(difference_type pos) { return reinterpret_cast<T*>(_union.direct) + pos; } const T* direct_ptr(difference_type pos) const { return reinterpret_cast<const T*>(_union.direct) + pos; } - T* indirect_ptr(difference_type pos) { return reinterpret_cast<T*>(_union.indirect) + pos; } - const T* indirect_ptr(difference_type pos) const { return reinterpret_cast<const T*>(_union.indirect) + pos; } + T* indirect_ptr(difference_type pos) { return reinterpret_cast<T*>(_union.indirect_contents.indirect) + pos; } + const T* indirect_ptr(difference_type pos) const { return reinterpret_cast<const T*>(_union.indirect_contents.indirect) + pos; } bool is_direct() const { return _size <= N; } void change_capacity(size_type new_capacity) { @@ -182,17 +182,17 @@ private: /* FIXME: Because malloc/realloc here won't call new_handler if allocation fails, assert success. These should instead use an allocator or new/delete so that handlers are called as necessary, but performance would be slightly degraded by doing so. */ - _union.indirect = static_cast<char*>(realloc(_union.indirect, ((size_t)sizeof(T)) * new_capacity)); - assert(_union.indirect); - _union.capacity = new_capacity; + _union.indirect_contents.indirect = static_cast<char*>(realloc(_union.indirect_contents.indirect, ((size_t)sizeof(T)) * new_capacity)); + assert(_union.indirect_contents.indirect); + _union.indirect_contents.capacity = new_capacity; } else { char* new_indirect = static_cast<char*>(malloc(((size_t)sizeof(T)) * new_capacity)); assert(new_indirect); T* src = direct_ptr(0); T* dst = reinterpret_cast<T*>(new_indirect); memcpy(dst, src, size() * sizeof(T)); - _union.indirect = new_indirect; - _union.capacity = new_capacity; + _union.indirect_contents.indirect = new_indirect; + _union.indirect_contents.capacity = new_capacity; _size += N + 1; } } @@ -301,7 +301,7 @@ public: if (is_direct()) { return N; } else { - return _union.capacity; + return _union.indirect_contents.capacity; } } @@ -468,8 +468,8 @@ public: clear(); } if (!is_direct()) { - free(_union.indirect); - _union.indirect = nullptr; + free(_union.indirect_contents.indirect); + _union.indirect_contents.indirect = nullptr; } } @@ -521,7 +521,7 @@ public: if (is_direct()) { return 0; } else { - return ((size_t)(sizeof(T))) * _union.capacity; + return ((size_t)(sizeof(T))) * _union.indirect_contents.capacity; } } diff --git a/src/primitives/block.h b/src/primitives/block.h index 750d42efbc..fd8fc8b868 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -33,17 +33,7 @@ public: SetNull(); } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(this->nVersion); - READWRITE(hashPrevBlock); - READWRITE(hashMerkleRoot); - READWRITE(nTime); - READWRITE(nBits); - READWRITE(nNonce); - } + SERIALIZE_METHODS(CBlockHeader, obj) { READWRITE(obj.nVersion, obj.hashPrevBlock, obj.hashMerkleRoot, obj.nTime, obj.nBits, obj.nNonce); } void SetNull() { @@ -89,12 +79,10 @@ public: *(static_cast<CBlockHeader*>(this)) = header; } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CBlockHeader, *this); - READWRITE(vtx); + SERIALIZE_METHODS(CBlock, obj) + { + READWRITEAS(CBlockHeader, obj); + READWRITE(obj.vtx); } void SetNull() @@ -131,14 +119,12 @@ struct CBlockLocator explicit CBlockLocator(const std::vector<uint256>& vHaveIn) : vHave(vHaveIn) {} - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(CBlockLocator, obj) + { int nVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); - READWRITE(vHave); + READWRITE(obj.vHave); } void SetNull() diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 00ccbc32f9..4514db578a 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -26,13 +26,7 @@ public: COutPoint(): n(NULL_INDEX) { } COutPoint(const uint256& hashIn, uint32_t nIn): hash(hashIn), n(nIn) { } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(hash); - READWRITE(n); - } + SERIALIZE_METHODS(COutPoint, obj) { READWRITE(obj.hash, obj.n); } void SetNull() { hash.SetNull(); n = NULL_INDEX; } bool IsNull() const { return (hash.IsNull() && n == NULL_INDEX); } @@ -103,14 +97,7 @@ public: explicit CTxIn(COutPoint prevoutIn, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); CTxIn(uint256 hashPrevTx, uint32_t nOut, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(prevout); - READWRITE(scriptSig); - READWRITE(nSequence); - } + SERIALIZE_METHODS(CTxIn, obj) { READWRITE(obj.prevout, obj.scriptSig, obj.nSequence); } friend bool operator==(const CTxIn& a, const CTxIn& b) { @@ -143,13 +130,7 @@ public: CTxOut(const CAmount& nValueIn, CScript scriptPubKeyIn); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nValue); - READWRITE(scriptPubKey); - } + SERIALIZE_METHODS(CTxOut, obj) { READWRITE(obj.nValue, obj.scriptPubKey); } void SetNull() { @@ -324,8 +305,6 @@ public: // Return sum of txouts. CAmount GetValueOut() const; - // GetValueIn() is a method on CCoinsViewCache, because - // inputs must be known to compute value in. /** * Get the total transaction size in bytes, including witness data. diff --git a/src/protocol.cpp b/src/protocol.cpp index a3e844e35b..243111c449 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -40,6 +40,10 @@ const char *SENDCMPCT="sendcmpct"; const char *CMPCTBLOCK="cmpctblock"; const char *GETBLOCKTXN="getblocktxn"; const char *BLOCKTXN="blocktxn"; +const char *GETCFHEADERS="getcfheaders"; +const char *CFHEADERS="cfheaders"; +const char *GETCFCHECKPT="getcfcheckpt"; +const char *CFCHECKPT="cfcheckpt"; } // namespace NetMsgType /** All known message types. Keep this in the same order as the list of @@ -71,6 +75,10 @@ const static std::string allNetMessageTypes[] = { NetMsgType::CMPCTBLOCK, NetMsgType::GETBLOCKTXN, NetMsgType::BLOCKTXN, + NetMsgType::GETCFHEADERS, + NetMsgType::CFHEADERS, + NetMsgType::GETCFCHECKPT, + NetMsgType::CFCHECKPT, }; const static std::vector<std::string> allNetMessageTypesVec(allNetMessageTypes, allNetMessageTypes+ARRAYLEN(allNetMessageTypes)); @@ -143,24 +151,6 @@ void SetServiceFlagsIBDCache(bool state) { g_initial_block_download_completed = state; } - -CAddress::CAddress() : CService() -{ - Init(); -} - -CAddress::CAddress(CService ipIn, ServiceFlags nServicesIn) : CService(ipIn) -{ - Init(); - nServices = nServicesIn; -} - -void CAddress::Init() -{ - nServices = NODE_NONE; - nTime = 100000000; -} - CInv::CInv() { type = 0; diff --git a/src/protocol.h b/src/protocol.h index 6639ae2aac..9527dce960 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -46,16 +46,7 @@ public: std::string GetCommand() const; bool IsValid(const MessageStartChars& messageStart) const; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(pchMessageStart); - READWRITE(pchCommand); - READWRITE(nMessageSize); - READWRITE(pchChecksum); - } + SERIALIZE_METHODS(CMessageHeader, obj) { READWRITE(obj.pchMessageStart, obj.pchCommand, obj.nMessageSize, obj.pchChecksum); } char pchMessageStart[MESSAGE_START_SIZE]; char pchCommand[COMMAND_SIZE]; @@ -74,100 +65,100 @@ namespace NetMsgType { * receiving node at the beginning of a connection. * @see https://bitcoin.org/en/developer-reference#version */ -extern const char *VERSION; +extern const char* VERSION; /** * The verack message acknowledges a previously-received version message, * informing the connecting node that it can begin to send other messages. * @see https://bitcoin.org/en/developer-reference#verack */ -extern const char *VERACK; +extern const char* VERACK; /** * The addr (IP address) message relays connection information for peers on the * network. * @see https://bitcoin.org/en/developer-reference#addr */ -extern const char *ADDR; +extern const char* ADDR; /** * The inv message (inventory message) transmits one or more inventories of * objects known to the transmitting peer. * @see https://bitcoin.org/en/developer-reference#inv */ -extern const char *INV; +extern const char* INV; /** * The getdata message requests one or more data objects from another node. * @see https://bitcoin.org/en/developer-reference#getdata */ -extern const char *GETDATA; +extern const char* GETDATA; /** * The merkleblock message is a reply to a getdata message which requested a * block using the inventory type MSG_MERKLEBLOCK. * @since protocol version 70001 as described by BIP37. * @see https://bitcoin.org/en/developer-reference#merkleblock */ -extern const char *MERKLEBLOCK; +extern const char* MERKLEBLOCK; /** * The getblocks message requests an inv message that provides block header * hashes starting from a particular point in the block chain. * @see https://bitcoin.org/en/developer-reference#getblocks */ -extern const char *GETBLOCKS; +extern const char* GETBLOCKS; /** * The getheaders message requests a headers message that provides block * headers starting from a particular point in the block chain. * @since protocol version 31800. * @see https://bitcoin.org/en/developer-reference#getheaders */ -extern const char *GETHEADERS; +extern const char* GETHEADERS; /** * The tx message transmits a single transaction. * @see https://bitcoin.org/en/developer-reference#tx */ -extern const char *TX; +extern const char* TX; /** * The headers message sends one or more block headers to a node which * previously requested certain headers with a getheaders message. * @since protocol version 31800. * @see https://bitcoin.org/en/developer-reference#headers */ -extern const char *HEADERS; +extern const char* HEADERS; /** * The block message transmits a single serialized block. * @see https://bitcoin.org/en/developer-reference#block */ -extern const char *BLOCK; +extern const char* BLOCK; /** * The getaddr message requests an addr message from the receiving node, * preferably one with lots of IP addresses of other receiving nodes. * @see https://bitcoin.org/en/developer-reference#getaddr */ -extern const char *GETADDR; +extern const char* GETADDR; /** * The mempool message requests the TXIDs of transactions that the receiving * node has verified as valid but which have not yet appeared in a block. * @since protocol version 60002. * @see https://bitcoin.org/en/developer-reference#mempool */ -extern const char *MEMPOOL; +extern const char* MEMPOOL; /** * The ping message is sent periodically to help confirm that the receiving * peer is still connected. * @see https://bitcoin.org/en/developer-reference#ping */ -extern const char *PING; +extern const char* PING; /** * The pong message replies to a ping message, proving to the pinging node that * the ponging node is still alive. * @since protocol version 60001 as described by BIP31. * @see https://bitcoin.org/en/developer-reference#pong */ -extern const char *PONG; +extern const char* PONG; /** * The notfound message is a reply to a getdata message which requested an * object the receiving node does not have available for relay. * @since protocol version 70001. * @see https://bitcoin.org/en/developer-reference#notfound */ -extern const char *NOTFOUND; +extern const char* NOTFOUND; /** * The filterload message tells the receiving peer to filter all relayed * transactions and requested merkle blocks through the provided filter. @@ -176,7 +167,7 @@ extern const char *NOTFOUND; * 70011 as described by BIP111. * @see https://bitcoin.org/en/developer-reference#filterload */ -extern const char *FILTERLOAD; +extern const char* FILTERLOAD; /** * The filteradd message tells the receiving peer to add a single element to a * previously-set bloom filter, such as a new public key. @@ -185,7 +176,7 @@ extern const char *FILTERLOAD; * 70011 as described by BIP111. * @see https://bitcoin.org/en/developer-reference#filteradd */ -extern const char *FILTERADD; +extern const char* FILTERADD; /** * The filterclear message tells the receiving peer to remove a previously-set * bloom filter. @@ -194,20 +185,20 @@ extern const char *FILTERADD; * 70011 as described by BIP111. * @see https://bitcoin.org/en/developer-reference#filterclear */ -extern const char *FILTERCLEAR; +extern const char* FILTERCLEAR; /** * Indicates that a node prefers to receive new block announcements via a * "headers" message rather than an "inv". * @since protocol version 70012 as described by BIP130. * @see https://bitcoin.org/en/developer-reference#sendheaders */ -extern const char *SENDHEADERS; +extern const char* SENDHEADERS; /** * The feefilter message tells the receiving peer not to inv us any txs * which do not meet the specified min fee rate. * @since protocol version 70013 as described by BIP133 */ -extern const char *FEEFILTER; +extern const char* FEEFILTER; /** * Contains a 1-byte bool and 8-byte LE version number. * Indicates that a node is willing to provide blocks via "cmpctblock" messages. @@ -215,29 +206,54 @@ extern const char *FEEFILTER; * "cmpctblock" message rather than an "inv", depending on message contents. * @since protocol version 70014 as described by BIP 152 */ -extern const char *SENDCMPCT; +extern const char* SENDCMPCT; /** * Contains a CBlockHeaderAndShortTxIDs object - providing a header and * list of "short txids". * @since protocol version 70014 as described by BIP 152 */ -extern const char *CMPCTBLOCK; +extern const char* CMPCTBLOCK; /** * Contains a BlockTransactionsRequest * Peer should respond with "blocktxn" message. * @since protocol version 70014 as described by BIP 152 */ -extern const char *GETBLOCKTXN; +extern const char* GETBLOCKTXN; /** * Contains a BlockTransactions. * Sent in response to a "getblocktxn" message. * @since protocol version 70014 as described by BIP 152 */ -extern const char *BLOCKTXN; -}; +extern const char* BLOCKTXN; +/** + * getcfheaders requests a compact filter header and the filter hashes for a + * range of blocks, which can then be used to reconstruct the filter headers + * for those blocks. + * Only available with service bit NODE_COMPACT_FILTERS as described by + * BIP 157 & 158. + */ +extern const char* GETCFHEADERS; +/** + * cfheaders is a response to a getcfheaders request containing a filter header + * and a vector of filter hashes for each subsequent block in the requested range. + */ +extern const char* CFHEADERS; +/** + * getcfcheckpt requests evenly spaced compact filter headers, enabling + * parallelized download and validation of the headers between them. + * Only available with service bit NODE_COMPACT_FILTERS as described by + * BIP 157 & 158. + */ +extern const char* GETCFCHECKPT; +/** + * cfcheckpt is a response to a getcfcheckpt request containing a vector of + * evenly spaced filter headers for blocks on the requested chain. + */ +extern const char* CFCHECKPT; +}; // namespace NetMsgType /* Get a vector of all valid message types (see above) */ -const std::vector<std::string> &getAllNetMessageTypes(); +const std::vector<std::string>& getAllNetMessageTypes(); /** nServices flags */ enum ServiceFlags : uint64_t { @@ -306,7 +322,8 @@ void SetServiceFlagsIBDCache(bool status); * == GetDesirableServiceFlags(services), ie determines whether the given * set of service flags are sufficient for a peer to be "relevant". */ -static inline bool HasAllDesirableServiceFlags(ServiceFlags services) { +static inline bool HasAllDesirableServiceFlags(ServiceFlags services) +{ return !(GetDesirableServiceFlags(services) & (~services)); } @@ -314,62 +331,55 @@ static inline bool HasAllDesirableServiceFlags(ServiceFlags services) { * Checks if a peer with the given service flags may be capable of having a * robust address-storage DB. */ -static inline bool MayHaveUsefulAddressDB(ServiceFlags services) { +static inline bool MayHaveUsefulAddressDB(ServiceFlags services) +{ return (services & NODE_NETWORK) || (services & NODE_NETWORK_LIMITED); } /** A CService with information about it as peer */ class CAddress : public CService { -public: - CAddress(); - explicit CAddress(CService ipIn, ServiceFlags nServicesIn); + static constexpr uint32_t TIME_INIT{100000000}; - void Init(); - - ADD_SERIALIZE_METHODS; +public: + CAddress() : CService{} {}; + explicit CAddress(CService ipIn, ServiceFlags nServicesIn) : CService{ipIn}, nServices{nServicesIn} {}; - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CAddress, obj) { - if (ser_action.ForRead()) - Init(); + SER_READ(obj, obj.nTime = TIME_INIT); int nVersion = s.GetVersion(); - if (s.GetType() & SER_DISK) + if (s.GetType() & SER_DISK) { READWRITE(nVersion); + } if ((s.GetType() & SER_DISK) || - (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) - READWRITE(nTime); - uint64_t nServicesInt = nServices; - READWRITE(nServicesInt); - nServices = static_cast<ServiceFlags>(nServicesInt); - READWRITEAS(CService, *this); + (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) { + READWRITE(obj.nTime); + } + READWRITE(Using<CustomUintFormatter<8>>(obj.nServices)); + READWRITEAS(CService, obj); } - // TODO: make private (improves encapsulation) -public: - ServiceFlags nServices; - + ServiceFlags nServices{NODE_NONE}; // disk and network only - unsigned int nTime; + uint32_t nTime{TIME_INIT}; }; /** getdata message type flags */ const uint32_t MSG_WITNESS_FLAG = 1 << 30; -const uint32_t MSG_TYPE_MASK = 0xffffffff >> 2; +const uint32_t MSG_TYPE_MASK = 0xffffffff >> 2; /** getdata / inv message types. * These numbers are defined by the protocol. When adding a new value, be sure * to mention it in the respective BIP. */ -enum GetDataMsg -{ +enum GetDataMsg { UNDEFINED = 0, MSG_TX = 1, MSG_BLOCK = 2, // The following can only occur in getdata. Invs always use TX or BLOCK. - MSG_FILTERED_BLOCK = 3, //!< Defined in BIP37 - MSG_CMPCT_BLOCK = 4, //!< Defined in BIP152 + MSG_FILTERED_BLOCK = 3, //!< Defined in BIP37 + MSG_CMPCT_BLOCK = 4, //!< Defined in BIP152 MSG_WITNESS_BLOCK = MSG_BLOCK | MSG_WITNESS_FLAG, //!< Defined in BIP144 MSG_WITNESS_TX = MSG_TX | MSG_WITNESS_FLAG, //!< Defined in BIP144 MSG_FILTERED_WITNESS_BLOCK = MSG_FILTERED_BLOCK | MSG_WITNESS_FLAG, @@ -382,21 +392,13 @@ public: CInv(); CInv(int typeIn, const uint256& hashIn); - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(type); - READWRITE(hash); - } + SERIALIZE_METHODS(CInv, obj) { READWRITE(obj.type, obj.hash); } friend bool operator<(const CInv& a, const CInv& b); std::string GetCommand() const; std::string ToString() const; -public: int type; uint256 hash; }; diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index 1aaf33c6a4..0d3b08fe7d 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -35,7 +35,7 @@ public: } protected: - bool filterAcceptsRow(int row, const QModelIndex& parent) const + bool filterAcceptsRow(int row, const QModelIndex& parent) const override { auto model = sourceModel(); auto label = model->index(row, AddressTableModel::Label, parent); @@ -136,6 +136,8 @@ AddressBookPage::AddressBookPage(const PlatformStyle *platformStyle, Mode _mode, connect(ui->tableView, &QWidget::customContextMenuRequested, this, &AddressBookPage::contextualMenu); connect(ui->closeButton, &QPushButton::clicked, this, &QDialog::accept); + + GUIUtil::handleCloseWindowShortcut(this); } AddressBookPage::~AddressBookPage() diff --git a/src/qt/addressbookpage.h b/src/qt/addressbookpage.h index 6394d26801..3d303a6f68 100644 --- a/src/qt/addressbookpage.h +++ b/src/qt/addressbookpage.h @@ -45,7 +45,7 @@ public: const QString &getReturnValue() const { return returnValue; } public Q_SLOTS: - void done(int retval); + void done(int retval) override; private: Ui::AddressBookPage *ui; diff --git a/src/qt/addresstablemodel.h b/src/qt/addresstablemodel.h index 035f3e0571..97f673caf1 100644 --- a/src/qt/addresstablemodel.h +++ b/src/qt/addresstablemodel.h @@ -52,14 +52,14 @@ public: /** @name Methods overridden from QAbstractTableModel @{*/ - int rowCount(const QModelIndex &parent) const; - int columnCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; - bool setData(const QModelIndex &index, const QVariant &value, int role); - QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex &parent) const; - bool removeRows(int row, int count, const QModelIndex &parent = QModelIndex()); - Qt::ItemFlags flags(const QModelIndex &index) const; + int rowCount(const QModelIndex &parent) const override; + int columnCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; + bool setData(const QModelIndex &index, const QVariant &value, int role) override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + QModelIndex index(int row, int column, const QModelIndex &parent) const override; + bool removeRows(int row, int count, const QModelIndex &parent = QModelIndex()) override; + Qt::ItemFlags flags(const QModelIndex &index) const override; /*@}*/ /* Add an address to the model. diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index 67e7704551..3d1963b6e6 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -10,6 +10,7 @@ #include <qt/forms/ui_askpassphrasedialog.h> #include <qt/guiconstants.h> +#include <qt/guiutil.h> #include <qt/walletmodel.h> #include <support/allocators/secure.h> @@ -75,6 +76,8 @@ AskPassphraseDialog::AskPassphraseDialog(Mode _mode, QWidget *parent, SecureStri connect(ui->passEdit1, &QLineEdit::textChanged, this, &AskPassphraseDialog::textChanged); connect(ui->passEdit2, &QLineEdit::textChanged, this, &AskPassphraseDialog::textChanged); connect(ui->passEdit3, &QLineEdit::textChanged, this, &AskPassphraseDialog::textChanged); + + GUIUtil::handleCloseWindowShortcut(this); } AskPassphraseDialog::~AskPassphraseDialog() diff --git a/src/qt/askpassphrasedialog.h b/src/qt/askpassphrasedialog.h index 20fc5045ae..9557e72936 100644 --- a/src/qt/askpassphrasedialog.h +++ b/src/qt/askpassphrasedialog.h @@ -32,7 +32,7 @@ public: explicit AskPassphraseDialog(Mode mode, QWidget *parent, SecureString* passphrase_out = nullptr); ~AskPassphraseDialog(); - void accept(); + void accept() override; void setModel(WalletModel *model); @@ -49,8 +49,8 @@ private Q_SLOTS: void toggleShowPassword(bool); protected: - bool event(QEvent *event); - bool eventFilter(QObject *object, QEvent *event); + bool event(QEvent *event) override; + bool eventFilter(QObject *object, QEvent *event) override; }; #endif // BITCOIN_QT_ASKPASSPHRASEDIALOG_H diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index 88d5b5811b..57f559fc14 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -56,16 +56,17 @@ public: /** @name Methods overridden from QAbstractTableModel @{*/ - int rowCount(const QModelIndex &parent) const; - int columnCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; - QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex &parent) const; - Qt::ItemFlags flags(const QModelIndex &index) const; - void sort(int column, Qt::SortOrder order); - bool shouldShow(); + int rowCount(const QModelIndex &parent) const override; + int columnCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + QModelIndex index(int row, int column, const QModelIndex &parent) const override; + Qt::ItemFlags flags(const QModelIndex &index) const override; + void sort(int column, Qt::SortOrder order) override; /*@}*/ + bool shouldShow(); + public Q_SLOTS: void refresh(); diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 8939b566f7..6fdf6322ff 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -34,6 +34,7 @@ #include <uint256.h> #include <util/system.h> #include <util/threadnames.h> +#include <validation.h> #include <memory> @@ -61,6 +62,7 @@ Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin); // Declare meta types used for QMetaObject::invokeMethod Q_DECLARE_METATYPE(bool*) Q_DECLARE_METATYPE(CAmount) +Q_DECLARE_METATYPE(SynchronizationState) Q_DECLARE_METATYPE(uint256) static QString GetLangTerritory() @@ -435,6 +437,7 @@ int GuiMain(int argc, char* argv[]) // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection qRegisterMetaType<bool*>(); + qRegisterMetaType<SynchronizationState>(); #ifdef ENABLE_WALLET qRegisterMetaType<WalletModel*>(); #endif diff --git a/src/qt/bitcoinaddressvalidator.h b/src/qt/bitcoinaddressvalidator.h index 30d4a26d0e..52c06828a3 100644 --- a/src/qt/bitcoinaddressvalidator.h +++ b/src/qt/bitcoinaddressvalidator.h @@ -17,7 +17,7 @@ class BitcoinAddressEntryValidator : public QValidator public: explicit BitcoinAddressEntryValidator(QObject *parent); - State validate(QString &input, int &pos) const; + State validate(QString &input, int &pos) const override; }; /** Bitcoin address widget validator, checks for a valid bitcoin address. @@ -29,7 +29,7 @@ class BitcoinAddressCheckValidator : public QValidator public: explicit BitcoinAddressCheckValidator(QObject *parent); - State validate(QString &input, int &pos) const; + State validate(QString &input, int &pos) const override; }; #endif // BITCOIN_QT_BITCOINADDRESSVALIDATOR_H diff --git a/src/qt/bitcoinamountfield.cpp b/src/qt/bitcoinamountfield.cpp index 7acc82370f..4c57f1e352 100644 --- a/src/qt/bitcoinamountfield.cpp +++ b/src/qt/bitcoinamountfield.cpp @@ -31,7 +31,7 @@ public: connect(lineEdit(), &QLineEdit::textEdited, this, &AmountSpinBox::valueChanged); } - QValidator::State validate(QString &text, int &pos) const + QValidator::State validate(QString &text, int &pos) const override { if(text.isEmpty()) return QValidator::Intermediate; @@ -41,7 +41,7 @@ public: return valid ? QValidator::Intermediate : QValidator::Invalid; } - void fixup(QString &input) const + void fixup(QString &input) const override { bool valid; CAmount val; @@ -87,7 +87,7 @@ public: m_max_amount = value; } - void stepBy(int steps) + void stepBy(int steps) override { bool valid = false; CAmount val = value(&valid); @@ -114,7 +114,7 @@ public: singleStep = step; } - QSize minimumSizeHint() const + QSize minimumSizeHint() const override { if(cachedMinimumSizeHint.isEmpty()) { @@ -175,7 +175,7 @@ private: } protected: - bool event(QEvent *event) + bool event(QEvent *event) override { if (event->type() == QEvent::KeyPress || event->type() == QEvent::KeyRelease) { @@ -190,7 +190,7 @@ protected: return QAbstractSpinBox::event(event); } - StepEnabled stepEnabled() const + StepEnabled stepEnabled() const override { if (isReadOnly()) // Disable steps when AmountSpinBox is read-only return StepNone; diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index 2db6b65f2c..d3e61aac29 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -70,7 +70,7 @@ Q_SIGNALS: protected: /** Intercept focus-in event and ',' key presses */ - bool eventFilter(QObject *object, QEvent *event); + bool eventFilter(QObject *object, QEvent *event) override; private: AmountSpinBox *amount; diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 321812c1f0..6192013e5f 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -36,6 +36,8 @@ #include <interfaces/node.h> #include <ui_interface.h> #include <util/system.h> +#include <util/translation.h> +#include <validation.h> #include <QAction> #include <QApplication> @@ -211,6 +213,8 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty #ifdef Q_OS_MAC m_app_nap_inhibitor = new CAppNapInhibitor; #endif + + GUIUtil::handleCloseWindowShortcut(this); } BitcoinGUI::~BitcoinGUI() @@ -573,7 +577,7 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) connect(_clientModel, &ClientModel::networkActiveChanged, this, &BitcoinGUI::setNetworkActive); modalOverlay->setKnownBestHeight(_clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(_clientModel->getHeaderTipTime())); - setNumBlocks(m_node.getNumBlocks(), QDateTime::fromTime_t(m_node.getLastBlockTime()), m_node.getVerificationProgress(), false); + setNumBlocks(m_node.getNumBlocks(), QDateTime::fromTime_t(m_node.getLastBlockTime()), m_node.getVerificationProgress(), false, SynchronizationState::INIT_DOWNLOAD); connect(_clientModel, &ClientModel::numBlocksChanged, this, &BitcoinGUI::setNumBlocks); // Receive and report messages from client model @@ -932,11 +936,15 @@ void BitcoinGUI::openOptionsDialogWithTab(OptionsDialog::Tab tab) dlg.exec(); } -void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool header) +void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool header, SynchronizationState sync_state) { // Disabling macOS App Nap on initial sync, disk and reindex operations. #ifdef Q_OS_MAC - (m_node.isInitialBlockDownload() || m_node.getReindex() || m_node.getImporting()) ? m_app_nap_inhibitor->disableAppNap() : m_app_nap_inhibitor->enableAppNap(); + if (sync_state == SynchronizationState::POST_INIT) { + m_app_nap_inhibitor->enableAppNap(); + } else { + m_app_nap_inhibitor->disableAppNap(); + } #endif if (modalOverlay) @@ -1046,7 +1054,7 @@ void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVer progressBar->setToolTip(tooltip); } -void BitcoinGUI::message(const QString& title, QString message, unsigned int style, bool* ret) +void BitcoinGUI::message(const QString& title, QString message, unsigned int style, bool* ret, const QString& detailed_message) { // Default title. On macOS, the window title is ignored (as required by the macOS Guidelines). QString strTitle{PACKAGE_NAME}; @@ -1100,6 +1108,7 @@ void BitcoinGUI::message(const QString& title, QString message, unsigned int sty showNormalIfMinimized(); QMessageBox mBox(static_cast<QMessageBox::Icon>(nMBoxIcon), strTitle, message, buttons, this); mBox.setTextFormat(Qt::PlainText); + mBox.setDetailedText(detailed_message); int r = mBox.exec(); if (ret != nullptr) *ret = r == QMessageBox::Ok; @@ -1251,7 +1260,7 @@ void BitcoinGUI::setEncryptionStatus(int status) labelWalletEncryptionIcon->setToolTip(tr("Wallet is <b>encrypted</b> and currently <b>unlocked</b>")); encryptWalletAction->setChecked(true); changePassphraseAction->setEnabled(true); - encryptWalletAction->setEnabled(false); // TODO: decrypt currently not supported + encryptWalletAction->setEnabled(false); break; case WalletModel::Locked: labelWalletEncryptionIcon->show(); @@ -1259,7 +1268,7 @@ void BitcoinGUI::setEncryptionStatus(int status) labelWalletEncryptionIcon->setToolTip(tr("Wallet is <b>encrypted</b> and currently <b>locked</b>")); encryptWalletAction->setChecked(true); changePassphraseAction->setEnabled(true); - encryptWalletAction->setEnabled(false); // TODO: decrypt currently not supported + encryptWalletAction->setEnabled(false); break; } } @@ -1375,20 +1384,27 @@ void BitcoinGUI::showModalOverlay() modalOverlay->toggleVisibility(); } -static bool ThreadSafeMessageBox(BitcoinGUI* gui, const std::string& message, const std::string& caption, unsigned int style) +static bool ThreadSafeMessageBox(BitcoinGUI* gui, const bilingual_str& message, const std::string& caption, unsigned int style) { bool modal = (style & CClientUIInterface::MODAL); // The SECURE flag has no effect in the Qt GUI. // bool secure = (style & CClientUIInterface::SECURE); style &= ~CClientUIInterface::SECURE; bool ret = false; + + QString detailed_message; // This is original message, in English, for googling and referencing. + if (message.original != message.translated) { + detailed_message = BitcoinGUI::tr("Original message:") + "\n" + QString::fromStdString(message.original); + } + // In case of modal message, use blocking connection to wait for user to click a button bool invoked = QMetaObject::invokeMethod(gui, "message", modal ? GUIUtil::blockingGUIThreadConnection() : Qt::QueuedConnection, Q_ARG(QString, QString::fromStdString(caption)), - Q_ARG(QString, QString::fromStdString(message)), + Q_ARG(QString, QString::fromStdString(message.translated)), Q_ARG(unsigned int, style), - Q_ARG(bool*, &ret)); + Q_ARG(bool*, &ret), + Q_ARG(QString, detailed_message)); assert(invoked); return ret; } diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 7326e2941a..c0198dd168 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -38,6 +38,7 @@ class WalletFrame; class WalletModel; class HelpMessageDialog; class ModalOverlay; +enum class SynchronizationState; namespace interfaces { class Handler; @@ -101,12 +102,12 @@ public: bool isPrivacyModeActivated() const; protected: - void changeEvent(QEvent *e); - void closeEvent(QCloseEvent *event); - void showEvent(QShowEvent *event); - void dragEnterEvent(QDragEnterEvent *event); - void dropEvent(QDropEvent *event); - bool eventFilter(QObject *object, QEvent *event); + void changeEvent(QEvent *e) override; + void closeEvent(QCloseEvent *event) override; + void showEvent(QShowEvent *event) override; + void dragEnterEvent(QDragEnterEvent *event) override; + void dropEvent(QDropEvent *event) override; + bool eventFilter(QObject *object, QEvent *event) override; private: interfaces::Node& m_node; @@ -217,16 +218,17 @@ public Q_SLOTS: /** Set network state shown in the UI */ void setNetworkActive(bool networkActive); /** Set number of blocks and last block date shown in the UI */ - void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers); + void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers, SynchronizationState sync_state); /** Notify the user of an event from the core network or transaction handling code. - @param[in] title the message box / notification title - @param[in] message the displayed text - @param[in] style modality and style definitions (icon and used buttons - buttons only for message boxes) - @see CClientUIInterface::MessageBoxFlags - @param[in] ret pointer to a bool that will be modified to whether Ok was clicked (modal only) + @param[in] title the message box / notification title + @param[in] message the displayed text + @param[in] style modality and style definitions (icon and used buttons - buttons only for message boxes) + @see CClientUIInterface::MessageBoxFlags + @param[in] ret pointer to a bool that will be modified to whether Ok was clicked (modal only) + @param[in] detailed_message the text to be displayed in the details area */ - void message(const QString& title, QString message, unsigned int style, bool* ret = nullptr); + void message(const QString& title, QString message, unsigned int style, bool* ret = nullptr, const QString& detailed_message = QString()); #ifdef ENABLE_WALLET void setCurrentWallet(WalletModel* wallet_model); @@ -328,7 +330,7 @@ public: protected: /** So that it responds to left-button clicks */ - void mousePressEvent(QMouseEvent *event); + void mousePressEvent(QMouseEvent *event) override; private: OptionsModel *optionsModel; diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index 5b8c351d73..dac5484393 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -92,8 +92,8 @@ public: /** Unit identifier */ UnitRole = Qt::UserRole }; - int rowCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; + int rowCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; ///@} static QString removeSpaces(QString text) diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index b94fcc9865..159b0d3df3 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -15,6 +15,7 @@ #include <net.h> #include <netbase.h> #include <util/system.h> +#include <validation.h> #include <stdint.h> @@ -234,17 +235,8 @@ static void BannedListChanged(ClientModel *clientmodel) assert(invoked); } -static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, int height, int64_t blockTime, double verificationProgress, bool fHeader) +static void BlockTipChanged(ClientModel* clientmodel, SynchronizationState sync_state, int height, int64_t blockTime, double verificationProgress, bool fHeader) { - // lock free async UI updates in case we have a new block tip - // during initial sync, only update the UI if the last update - // was > 250ms (MODEL_UPDATE_DELAY) ago - int64_t now = 0; - if (initialSync) - now = GetTimeMillis(); - - int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; - if (fHeader) { // cache best headers time and height to reduce future cs_main locks clientmodel->cachedBestHeaderHeight = height; @@ -253,17 +245,22 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, int heig clientmodel->m_cached_num_blocks = height; } - // During initial sync, block notifications, and header notifications from reindexing are both throttled. - if (!initialSync || (fHeader && !clientmodel->node().getReindex()) || now - nLastUpdateNotification > MODEL_UPDATE_DELAY) { - //pass an async signal to the UI thread - bool invoked = QMetaObject::invokeMethod(clientmodel, "numBlocksChanged", Qt::QueuedConnection, - Q_ARG(int, height), - Q_ARG(QDateTime, QDateTime::fromTime_t(blockTime)), - Q_ARG(double, verificationProgress), - Q_ARG(bool, fHeader)); - assert(invoked); - nLastUpdateNotification = now; + // Throttle GUI notifications about (a) blocks during initial sync, and (b) both blocks and headers during reindex. + const bool throttle = (sync_state != SynchronizationState::POST_INIT && !fHeader) || sync_state == SynchronizationState::INIT_REINDEX; + const int64_t now = throttle ? GetTimeMillis() : 0; + int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; + if (throttle && now < nLastUpdateNotification + MODEL_UPDATE_DELAY) { + return; } + + bool invoked = QMetaObject::invokeMethod(clientmodel, "numBlocksChanged", Qt::QueuedConnection, + Q_ARG(int, height), + Q_ARG(QDateTime, QDateTime::fromTime_t(blockTime)), + Q_ARG(double, verificationProgress), + Q_ARG(bool, fHeader), + Q_ARG(SynchronizationState, sync_state)); + assert(invoked); + nLastUpdateNotification = now; } void ClientModel::subscribeToCoreSignals() diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 7ac4120a8f..ace77f5972 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -12,10 +12,10 @@ #include <memory> class BanTableModel; +class CBlockIndex; class OptionsModel; class PeerTableModel; - -class CBlockIndex; +enum class SynchronizationState; namespace interfaces { class Handler; @@ -100,7 +100,7 @@ private: Q_SIGNALS: void numConnectionsChanged(int count); - void numBlocksChanged(int count, const QDateTime& blockDate, double nVerificationProgress, bool header); + void numBlocksChanged(int count, const QDateTime& blockDate, double nVerificationProgress, bool header, SynchronizationState sync_state); void mempoolSizeChanged(long count, size_t mempoolSizeInBytes); void networkActiveChanged(bool networkActive); void alertsChanged(const QString &warnings); diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 1889f5e056..db77c17df0 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -41,10 +41,11 @@ bool CCoinControlWidgetItem::operator<(const QTreeWidgetItem &other) const { return QTreeWidgetItem::operator<(other); } -CoinControlDialog::CoinControlDialog(const PlatformStyle *_platformStyle, QWidget *parent) : +CoinControlDialog::CoinControlDialog(CCoinControl& coin_control, WalletModel* _model, const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent), ui(new Ui::CoinControlDialog), - model(nullptr), + m_coin_control(coin_control), + model(_model), platformStyle(_platformStyle) { ui->setupUi(this); @@ -134,6 +135,15 @@ CoinControlDialog::CoinControlDialog(const PlatformStyle *_platformStyle, QWidge ui->radioTreeMode->click(); if (settings.contains("nCoinControlSortColumn") && settings.contains("nCoinControlSortOrder")) sortView(settings.value("nCoinControlSortColumn").toInt(), (static_cast<Qt::SortOrder>(settings.value("nCoinControlSortOrder").toInt()))); + + GUIUtil::handleCloseWindowShortcut(this); + + if(_model->getOptionsModel() && _model->getAddressTableModel()) + { + updateView(); + updateLabelLocked(); + CoinControlDialog::updateLabels(m_coin_control, _model, this); + } } CoinControlDialog::~CoinControlDialog() @@ -146,18 +156,6 @@ CoinControlDialog::~CoinControlDialog() delete ui; } -void CoinControlDialog::setModel(WalletModel *_model) -{ - this->model = _model; - - if(_model && _model->getOptionsModel() && _model->getAddressTableModel()) - { - updateView(); - updateLabelLocked(); - CoinControlDialog::updateLabels(_model, this); - } -} - // ok button void CoinControlDialog::buttonBoxClicked(QAbstractButton* button) { @@ -183,8 +181,8 @@ void CoinControlDialog::buttonSelectAllClicked() ui->treeWidget->topLevelItem(i)->setCheckState(COLUMN_CHECKBOX, state); ui->treeWidget->setEnabled(true); if (state == Qt::Unchecked) - coinControl()->UnSelectAll(); // just to be sure - CoinControlDialog::updateLabels(model, this); + m_coin_control.UnSelectAll(); // just to be sure + CoinControlDialog::updateLabels(m_coin_control, model, this); } // context menu @@ -369,15 +367,15 @@ void CoinControlDialog::viewItemChanged(QTreeWidgetItem* item, int column) COutPoint outpt(uint256S(item->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), item->data(COLUMN_ADDRESS, VOutRole).toUInt()); if (item->checkState(COLUMN_CHECKBOX) == Qt::Unchecked) - coinControl()->UnSelect(outpt); + m_coin_control.UnSelect(outpt); else if (item->isDisabled()) // locked (this happens if "check all" through parent node) item->setCheckState(COLUMN_CHECKBOX, Qt::Unchecked); else - coinControl()->Select(outpt); + m_coin_control.Select(outpt); // selection changed -> update labels if (ui->treeWidget->isEnabled()) // do not update on every click for (un)select all - CoinControlDialog::updateLabels(model, this); + CoinControlDialog::updateLabels(m_coin_control, model, this); } } @@ -394,7 +392,7 @@ void CoinControlDialog::updateLabelLocked() else ui->labelLocked->setVisible(false); } -void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) +void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel *model, QDialog* dialog) { if (!model) return; @@ -426,7 +424,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) bool fWitness = false; std::vector<COutPoint> vCoinControl; - coinControl()->ListSelected(vCoinControl); + m_coin_control.ListSelected(vCoinControl); size_t i = 0; for (const auto& out : model->wallet().getCoins(vCoinControl)) { @@ -437,7 +435,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) const COutPoint& outpt = vCoinControl[i++]; if (out.is_spent) { - coinControl()->UnSelect(outpt); + m_coin_control.UnSelect(outpt); continue; } @@ -490,7 +488,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) nBytes -= 34; // Fee - nPayFee = model->wallet().getMinimumFee(nBytes, *coinControl(), nullptr /* returned_target */, nullptr /* reason */); + nPayFee = model->wallet().getMinimumFee(nBytes, m_coin_control, nullptr /* returned_target */, nullptr /* reason */); if (nPayAmount > 0) { @@ -582,12 +580,6 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) label->setVisible(nChange < 0); } -CCoinControl* CoinControlDialog::coinControl() -{ - static CCoinControl coin_control; - return &coin_control; -} - void CoinControlDialog::updateView() { if (!model || !model->getOptionsModel() || !model->getAddressTableModel()) @@ -604,8 +596,7 @@ void CoinControlDialog::updateView() int nDisplayUnit = model->getOptionsModel()->getDisplayUnit(); for (const auto& coins : model->wallet().listCoins()) { - CCoinControlWidgetItem *itemWalletAddress = new CCoinControlWidgetItem(); - itemWalletAddress->setCheckState(COLUMN_CHECKBOX, Qt::Unchecked); + CCoinControlWidgetItem* itemWalletAddress{nullptr}; QString sWalletAddress = QString::fromStdString(EncodeDestination(coins.first)); QString sWalletLabel = model->getAddressTableModel()->labelForAddress(sWalletAddress); if (sWalletLabel.isEmpty()) @@ -614,7 +605,7 @@ void CoinControlDialog::updateView() if (treeMode) { // wallet address - ui->treeWidget->addTopLevelItem(itemWalletAddress); + itemWalletAddress = new CCoinControlWidgetItem(ui->treeWidget); itemWalletAddress->setFlags(flgTristate); itemWalletAddress->setCheckState(COLUMN_CHECKBOX, Qt::Unchecked); @@ -688,13 +679,13 @@ void CoinControlDialog::updateView() // disable locked coins if (model->wallet().isLockedCoin(output)) { - coinControl()->UnSelect(output); // just to be sure + m_coin_control.UnSelect(output); // just to be sure itemOutput->setDisabled(true); itemOutput->setIcon(COLUMN_CHECKBOX, platformStyle->SingleColorIcon(":/icons/lock_closed")); } // set checkbox - if (coinControl()->IsSelected(output)) + if (m_coin_control.IsSelected(output)) itemOutput->setCheckState(COLUMN_CHECKBOX, Qt::Checked); } diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h index efc06a7656..3de7fd6d54 100644 --- a/src/qt/coincontroldialog.h +++ b/src/qt/coincontroldialog.h @@ -31,10 +31,9 @@ class CCoinControlWidgetItem : public QTreeWidgetItem { public: explicit CCoinControlWidgetItem(QTreeWidget *parent, int type = Type) : QTreeWidgetItem(parent, type) {} - explicit CCoinControlWidgetItem(int type = Type) : QTreeWidgetItem(type) {} explicit CCoinControlWidgetItem(QTreeWidgetItem *parent, int type = Type) : QTreeWidgetItem(parent, type) {} - bool operator<(const QTreeWidgetItem &other) const; + bool operator<(const QTreeWidgetItem &other) const override; }; @@ -43,20 +42,18 @@ class CoinControlDialog : public QDialog Q_OBJECT public: - explicit CoinControlDialog(const PlatformStyle *platformStyle, QWidget *parent = nullptr); + explicit CoinControlDialog(CCoinControl& coin_control, WalletModel* model, const PlatformStyle *platformStyle, QWidget *parent = nullptr); ~CoinControlDialog(); - void setModel(WalletModel *model); - // static because also called from sendcoinsdialog - static void updateLabels(WalletModel*, QDialog*); + static void updateLabels(CCoinControl& m_coin_control, WalletModel*, QDialog*); static QList<CAmount> payAmounts; - static CCoinControl *coinControl(); static bool fSubtractFeeFromAmount; private: Ui::CoinControlDialog *ui; + CCoinControl& m_coin_control; WalletModel *model; int sortColumn; Qt::SortOrder sortOrder; diff --git a/src/qt/coincontroltreewidget.h b/src/qt/coincontroltreewidget.h index 86af555385..ac03a409c1 100644 --- a/src/qt/coincontroltreewidget.h +++ b/src/qt/coincontroltreewidget.h @@ -16,7 +16,7 @@ public: explicit CoinControlTreeWidget(QWidget *parent = nullptr); protected: - virtual void keyPressEvent(QKeyEvent *event); + virtual void keyPressEvent(QKeyEvent *event) override; }; #endif // BITCOIN_QT_COINCONTROLTREEWIDGET_H diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index 4711412ac4..e0af9a20a8 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -43,6 +43,8 @@ EditAddressDialog::EditAddressDialog(Mode _mode, QWidget *parent) : GUIUtil::ItemDelegate* delegate = new GUIUtil::ItemDelegate(mapper); connect(delegate, &GUIUtil::ItemDelegate::keyEscapePressed, this, &EditAddressDialog::reject); mapper->setItemDelegate(delegate); + + GUIUtil::handleCloseWindowShortcut(this); } EditAddressDialog::~EditAddressDialog() diff --git a/src/qt/editaddressdialog.h b/src/qt/editaddressdialog.h index 4d0e0709be..3be63156fd 100644 --- a/src/qt/editaddressdialog.h +++ b/src/qt/editaddressdialog.h @@ -40,7 +40,7 @@ public: void setAddress(const QString &address); public Q_SLOTS: - void accept(); + void accept() override; private: bool saveCurrentRow(); diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index a54b65961c..af86fe5d27 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -54,6 +54,7 @@ #include <QSettings> #include <QSize> #include <QString> +#include <QShortcut> #include <QTextDocument> // for Qt::mightBeRichText #include <QThread> #include <QUrlQuery> @@ -378,6 +379,11 @@ void bringToFront(QWidget* w) } } +void handleCloseWindowShortcut(QWidget* w) +{ + QObject::connect(new QShortcut(QKeySequence(Qt::CTRL + Qt::Key_W), w), &QShortcut::activated, w, &QWidget::close); +} + void openDebugLogfile() { fs::path pathDebug = GetDataDir() / "debug.log"; diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index e571262443..8b9fca4fb1 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -124,6 +124,9 @@ namespace GUIUtil // Activate, show and raise the widget void bringToFront(QWidget* w); + // Set shortcut to close window + void handleCloseWindowShortcut(QWidget* w); + // Open debug.log void openDebugLogfile(); @@ -142,7 +145,7 @@ namespace GUIUtil explicit ToolTipToRichTextFilter(int size_threshold, QObject *parent = nullptr); protected: - bool eventFilter(QObject *obj, QEvent *evt); + bool eventFilter(QObject *obj, QEvent *evt) override; private: int size_threshold; @@ -224,7 +227,7 @@ namespace GUIUtil */ void clicked(const QPoint& point); protected: - void mouseReleaseEvent(QMouseEvent *event); + void mouseReleaseEvent(QMouseEvent *event) override; }; class ClickableProgressBar : public QProgressBar @@ -237,7 +240,7 @@ namespace GUIUtil */ void clicked(const QPoint& point); protected: - void mouseReleaseEvent(QMouseEvent *event); + void mouseReleaseEvent(QMouseEvent *event) override; }; typedef ClickableProgressBar ProgressBar; @@ -252,7 +255,7 @@ namespace GUIUtil void keyEscapePressed(); private: - bool eventFilter(QObject *object, QEvent *event); + bool eventFilter(QObject *object, QEvent *event) override; }; // Fix known bugs in QProgressDialog class. diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index 076ec30b58..a565d7d8f3 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -35,8 +35,8 @@ public Q_SLOTS: bool isLayerVisible() const { return layerIsVisible; } protected: - bool eventFilter(QObject * obj, QEvent * ev); - bool event(QEvent* ev); + bool eventFilter(QObject * obj, QEvent * ev) override; + bool event(QEvent* ev) override; private: Ui::ModalOverlay *ui; diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index b9dea2f8bf..9a3d43c2a6 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -15,6 +15,8 @@ OpenURIDialog::OpenURIDialog(QWidget *parent) : ui(new Ui::OpenURIDialog) { ui->setupUi(this); + + GUIUtil::handleCloseWindowShortcut(this); } OpenURIDialog::~OpenURIDialog() diff --git a/src/qt/openuridialog.h b/src/qt/openuridialog.h index 4b610f74d7..667af2ec75 100644 --- a/src/qt/openuridialog.h +++ b/src/qt/openuridialog.h @@ -22,7 +22,7 @@ public: QString getURI(); protected Q_SLOTS: - void accept(); + void accept() override; private: Ui::OpenURIDialog *ui; diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 8ee6c947e6..ae6aeb7709 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -135,6 +135,8 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : ui->minimizeToTray->setChecked(false); ui->minimizeToTray->setEnabled(false); } + + GUIUtil::handleCloseWindowShortcut(this); } OptionsDialog::~OptionsDialog() diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index 9bc1c8ae4f..568c8b6fd0 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -28,7 +28,7 @@ class ProxyAddressValidator : public QValidator public: explicit ProxyAddressValidator(QObject *parent); - State validate(QString &input, int &pos) const; + State validate(QString &input, int &pos) const override; }; /** Preferences dialog. */ diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index b3260349e7..6ca5ac9d75 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -68,9 +68,9 @@ public: void Init(bool resetSettings = false); void Reset(); - int rowCount(const QModelIndex & parent = QModelIndex()) const; - QVariant data(const QModelIndex & index, int role = Qt::DisplayRole) const; - bool setData(const QModelIndex & index, const QVariant & value, int role = Qt::EditRole); + int rowCount(const QModelIndex & parent = QModelIndex()) const override; + QVariant data(const QModelIndex & index, int role = Qt::DisplayRole) const override; + bool setData(const QModelIndex & index, const QVariant & value, int role = Qt::EditRole) override; /** Updates current unit in memory, settings and emits displayUnitChanged(newUnit) signal */ void setDisplayUnit(const QVariant &value); diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 1ab30fefc7..0af70f2735 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -37,7 +37,7 @@ public: } inline void paint(QPainter *painter, const QStyleOptionViewItem &option, - const QModelIndex &index ) const + const QModelIndex &index ) const override { painter->save(); @@ -101,7 +101,7 @@ public: painter->restore(); } - inline QSize sizeHint(const QStyleOptionViewItem &option, const QModelIndex &index) const + inline QSize sizeHint(const QStyleOptionViewItem &option, const QModelIndex &index) const override { return QSize(DECORATION_SIZE, DECORATION_SIZE); } diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index aa9a7327ba..154f4a7ea6 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -98,7 +98,7 @@ private Q_SLOTS: protected: // Constructor registers this on the parent QApplication to // receive QEvent::FileOpen and QEvent:Drop events - bool eventFilter(QObject *object, QEvent *event); + bool eventFilter(QObject *object, QEvent *event) override; private: bool saveURIs; // true during startup diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index 0e20b1d9dc..99de772ac0 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -68,13 +68,13 @@ public: /** @name Methods overridden from QAbstractTableModel @{*/ - int rowCount(const QModelIndex &parent) const; - int columnCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; - QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex &parent) const; - Qt::ItemFlags flags(const QModelIndex &index) const; - void sort(int column, Qt::SortOrder order); + int rowCount(const QModelIndex &parent) const override; + int columnCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + QModelIndex index(int row, int column, const QModelIndex &parent) const override; + Qt::ItemFlags flags(const QModelIndex &index) const override; + void sort(int column, Qt::SortOrder order) override; /*@}*/ public Q_SLOTS: diff --git a/src/qt/qrimagewidget.h b/src/qt/qrimagewidget.h index 345bb64092..cca598c2ce 100644 --- a/src/qt/qrimagewidget.h +++ b/src/qt/qrimagewidget.h @@ -35,8 +35,8 @@ public Q_SLOTS: void copyImage(); protected: - virtual void mousePressEvent(QMouseEvent *event); - virtual void contextMenuEvent(QContextMenuEvent *event); + virtual void mousePressEvent(QMouseEvent *event) override; + virtual void contextMenuEvent(QContextMenuEvent *event) override; private: QMenu *contextMenu; diff --git a/src/qt/qvalidatedlineedit.h b/src/qt/qvalidatedlineedit.h index 815d5e0040..2c72b2ecda 100644 --- a/src/qt/qvalidatedlineedit.h +++ b/src/qt/qvalidatedlineedit.h @@ -21,8 +21,8 @@ public: bool isValid(); protected: - void focusInEvent(QFocusEvent *evt); - void focusOutEvent(QFocusEvent *evt); + void focusInEvent(QFocusEvent *evt) override; + void focusOutEvent(QFocusEvent *evt) override; private: bool valid; diff --git a/src/qt/receivecoinsdialog.h b/src/qt/receivecoinsdialog.h index 5bca2f46e2..2f48cd58f0 100644 --- a/src/qt/receivecoinsdialog.h +++ b/src/qt/receivecoinsdialog.h @@ -46,11 +46,11 @@ public: public Q_SLOTS: void clear(); - void reject(); - void accept(); + void reject() override; + void accept() override; protected: - virtual void keyPressEvent(QKeyEvent *event); + virtual void keyPressEvent(QKeyEvent *event) override; private: Ui::ReceiveCoinsDialog *ui; @@ -61,7 +61,7 @@ private: QModelIndex selectedRow(); void copyColumnToClipboard(int column); - virtual void resizeEvent(QResizeEvent *event); + virtual void resizeEvent(QResizeEvent *event) override; private Q_SLOTS: void on_receiveButton_clicked(); diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index b4fae7d78d..30bd5c6a5a 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -30,6 +30,8 @@ ReceiveRequestDialog::ReceiveRequestDialog(QWidget *parent) : #endif connect(ui->btnSaveAs, &QPushButton::clicked, ui->lblQRCode, &QRImageWidget::saveImage); + + GUIUtil::handleCloseWindowShortcut(this); } ReceiveRequestDialog::~ReceiveRequestDialog() diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index 5e7f6acdc8..c0bd3461bb 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -24,19 +24,11 @@ public: QDateTime date; SendCoinsRecipient recipient; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - unsigned int nDate = date.toTime_t(); - - READWRITE(this->nVersion); - READWRITE(id); - READWRITE(nDate); - READWRITE(recipient); - - if (ser_action.ForRead()) - date = QDateTime::fromTime_t(nDate); + SERIALIZE_METHODS(RecentRequestEntry, obj) { + unsigned int date_timet; + SER_WRITE(obj, date_timet = obj.date.toTime_t()); + READWRITE(obj.nVersion, obj.id, date_timet, obj.recipient); + SER_READ(obj, obj.date = QDateTime::fromTime_t(date_timet)); } }; @@ -73,14 +65,15 @@ public: /** @name Methods overridden from QAbstractTableModel @{*/ - int rowCount(const QModelIndex &parent) const; - int columnCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; - bool setData(const QModelIndex &index, const QVariant &value, int role); - QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex &parent = QModelIndex()) const; - bool removeRows(int row, int count, const QModelIndex &parent = QModelIndex()); - Qt::ItemFlags flags(const QModelIndex &index) const; + int rowCount(const QModelIndex &parent) const override; + int columnCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; + bool setData(const QModelIndex &index, const QVariant &value, int role) override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + QModelIndex index(int row, int column, const QModelIndex &parent = QModelIndex()) const override; + bool removeRows(int row, int count, const QModelIndex &parent = QModelIndex()) override; + Qt::ItemFlags flags(const QModelIndex &index) const override; + void sort(int column, Qt::SortOrder order = Qt::AscendingOrder) override; /*@}*/ const RecentRequestEntry &entry(int row) const { return list[row]; } @@ -89,7 +82,6 @@ public: void addNewRequest(RecentRequestEntry &recipient); public Q_SLOTS: - void sort(int column, Qt::SortOrder order = Qt::AscendingOrder); void updateDisplayUnit(); private: diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 18a7e9b021..66f1c8fd9c 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -24,23 +24,21 @@ #include <univalue.h> #ifdef ENABLE_WALLET -#include <db_cxx.h> +#include <wallet/db.h> #include <wallet/wallet.h> #endif #include <QKeyEvent> #include <QMenu> #include <QMessageBox> -#include <QScrollBar> #include <QScreen> +#include <QScrollBar> #include <QSettings> +#include <QString> +#include <QStringList> #include <QTime> #include <QTimer> -#include <QStringList> -// TODO: add a scrollback limit, as there is currently none -// TODO: make it possible to filter out categories (esp debug messages when implemented) -// TODO: receive errors and debug messages through ClientModel const int CONSOLE_HISTORY = 50; const int INITIAL_TRAFFIC_GRAPH_MINS = 30; @@ -115,8 +113,8 @@ class QtRPCTimerInterface: public RPCTimerInterface { public: ~QtRPCTimerInterface() {} - const char *Name() { return "Qt"; } - RPCTimerBase* NewTimer(std::function<void()>& func, int64_t millis) + const char *Name() override { return "Qt"; } + RPCTimerBase* NewTimer(std::function<void()>& func, int64_t millis) override { return new QtRPCTimerBase(func, millis); } @@ -480,7 +478,7 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty // set library version labels #ifdef ENABLE_WALLET - ui->berkeleyDBVersion->setText(DbEnv::version(nullptr, nullptr, nullptr)); + ui->berkeleyDBVersion->setText(QString::fromStdString(BerkeleyDatabaseVersion())); #else ui->label_berkeleyDBVersion->hide(); ui->berkeleyDBVersion->hide(); @@ -498,6 +496,8 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty consoleFontSize = settings.value(fontSizeSettingsKey, QFontInfo(QFont()).pointSize()).toInt(); clear(); + + GUIUtil::handleCloseWindowShortcut(this); } RPCConsole::~RPCConsole() diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index f586d04022..de8e37cca2 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -71,8 +71,8 @@ public: QKeySequence tabShortcut(TabTypes tab_type) const; protected: - virtual bool eventFilter(QObject* obj, QEvent *event); - void keyPressEvent(QKeyEvent *); + virtual bool eventFilter(QObject* obj, QEvent *event) override; + void keyPressEvent(QKeyEvent *) override; private Q_SLOTS: void on_lineEdit_returnPressed(); @@ -83,9 +83,9 @@ private Q_SLOTS: void on_sldGraphRange_valueChanged(int value); /** update traffic statistics */ void updateTrafficStats(quint64 totalBytesIn, quint64 totalBytesOut); - void resizeEvent(QResizeEvent *event); - void showEvent(QShowEvent *event); - void hideEvent(QHideEvent *event); + void resizeEvent(QResizeEvent *event) override; + void showEvent(QShowEvent *event) override; + void hideEvent(QHideEvent *event) override; /** Show custom context menu on Peers tab */ void showPeersTableContextMenu(const QPoint& point); /** Show custom context menu on Bans tab */ diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 7b389a48d6..9e23fe78d8 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -57,6 +57,7 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui(new Ui::SendCoinsDialog), clientModel(nullptr), model(nullptr), + m_coin_control(new CCoinControl), fNewRecipientAllowed(true), fFeeMinimized(true), platformStyle(_platformStyle) @@ -259,14 +260,9 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa m_current_transaction = MakeUnique<WalletModelTransaction>(recipients); WalletModel::SendCoinsReturn prepareStatus; - // Always use a CCoinControl instance, use the CoinControlDialog instance if CoinControl has been enabled - CCoinControl ctrl; - if (model->getOptionsModel()->getCoinControlFeatures()) - ctrl = *CoinControlDialog::coinControl(); + updateCoinControlState(*m_coin_control); - updateCoinControlState(ctrl); - - prepareStatus = model->prepareTransaction(*m_current_transaction, ctrl); + prepareStatus = model->prepareTransaction(*m_current_transaction, *m_coin_control); // process prepareStatus and on error generate message shown to user processSendCoinsReturn(prepareStatus, @@ -454,7 +450,7 @@ void SendCoinsDialog::on_sendButton_clicked() } if (!send_failure) { accept(); - CoinControlDialog::coinControl()->UnSelectAll(); + m_coin_control->UnSelectAll(); coinControlUpdateLabels(); } fNewRecipientAllowed = true; @@ -466,7 +462,7 @@ void SendCoinsDialog::clear() m_current_transaction.reset(); // Clear coin control settings - CoinControlDialog::coinControl()->UnSelectAll(); + m_coin_control->UnSelectAll(); ui->checkBoxCoinControlChange->setChecked(false); ui->lineEditCoinControlChange->clear(); coinControlUpdateLabels(); @@ -689,17 +685,11 @@ void SendCoinsDialog::on_buttonMinimizeFee_clicked() void SendCoinsDialog::useAvailableBalance(SendCoinsEntry* entry) { - // Get CCoinControl instance if CoinControl is enabled or create a new one. - CCoinControl coin_control; - if (model->getOptionsModel()->getCoinControlFeatures()) { - coin_control = *CoinControlDialog::coinControl(); - } - // Include watch-only for wallets without private key - coin_control.fAllowWatchOnly = model->wallet().privateKeysDisabled(); + m_coin_control->fAllowWatchOnly = model->wallet().privateKeysDisabled(); // Calculate available amount to send. - CAmount amount = model->wallet().getAvailableBalance(coin_control); + CAmount amount = model->wallet().getAvailableBalance(*m_coin_control); for (int i = 0; i < ui->entries->count(); ++i) { SendCoinsEntry* e = qobject_cast<SendCoinsEntry*>(ui->entries->itemAt(i)->widget()); if (e && !e->isHidden() && e != entry) { @@ -758,12 +748,11 @@ void SendCoinsDialog::updateSmartFeeLabel() { if(!model || !model->getOptionsModel()) return; - CCoinControl coin_control; - updateCoinControlState(coin_control); - coin_control.m_feerate.reset(); // Explicitly use only fee estimation rate for smart fee labels + updateCoinControlState(*m_coin_control); + m_coin_control->m_feerate.reset(); // Explicitly use only fee estimation rate for smart fee labels int returned_target; FeeReason reason; - CFeeRate feeRate = CFeeRate(model->wallet().getMinimumFee(1000, coin_control, &returned_target, &reason)); + CFeeRate feeRate = CFeeRate(model->wallet().getMinimumFee(1000, *m_coin_control, &returned_target, &reason)); ui->labelSmartFee->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), feeRate.GetFeePerK()) + "/kB"); @@ -834,7 +823,7 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) ui->frameCoinControl->setVisible(checked); if (!checked && model) // coin control features disabled - CoinControlDialog::coinControl()->SetNull(); + m_coin_control->SetNull(); coinControlUpdateLabels(); } @@ -842,8 +831,7 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) // Coin Control: button inputs -> show actual coin control dialog void SendCoinsDialog::coinControlButtonClicked() { - CoinControlDialog dlg(platformStyle); - dlg.setModel(model); + CoinControlDialog dlg(*m_coin_control, model, platformStyle); dlg.exec(); coinControlUpdateLabels(); } @@ -853,7 +841,7 @@ void SendCoinsDialog::coinControlChangeChecked(int state) { if (state == Qt::Unchecked) { - CoinControlDialog::coinControl()->destChange = CNoDestination(); + m_coin_control->destChange = CNoDestination(); ui->labelCoinControlChangeLabel->clear(); } else @@ -869,7 +857,7 @@ void SendCoinsDialog::coinControlChangeEdited(const QString& text) if (model && model->getAddressTableModel()) { // Default to no change address until verified - CoinControlDialog::coinControl()->destChange = CNoDestination(); + m_coin_control->destChange = CNoDestination(); ui->labelCoinControlChangeLabel->setStyleSheet("QLabel{color:red;}"); const CTxDestination dest = DecodeDestination(text.toStdString()); @@ -892,7 +880,7 @@ void SendCoinsDialog::coinControlChangeEdited(const QString& text) QMessageBox::Yes | QMessageBox::Cancel, QMessageBox::Cancel); if(btnRetVal == QMessageBox::Yes) - CoinControlDialog::coinControl()->destChange = dest; + m_coin_control->destChange = dest; else { ui->lineEditCoinControlChange->setText(""); @@ -911,7 +899,7 @@ void SendCoinsDialog::coinControlChangeEdited(const QString& text) else ui->labelCoinControlChangeLabel->setText(tr("(no label)")); - CoinControlDialog::coinControl()->destChange = dest; + m_coin_control->destChange = dest; } } } @@ -923,7 +911,7 @@ void SendCoinsDialog::coinControlUpdateLabels() if (!model || !model->getOptionsModel()) return; - updateCoinControlState(*CoinControlDialog::coinControl()); + updateCoinControlState(*m_coin_control); // set pay amounts CoinControlDialog::payAmounts.clear(); @@ -941,10 +929,10 @@ void SendCoinsDialog::coinControlUpdateLabels() } } - if (CoinControlDialog::coinControl()->HasSelected()) + if (m_coin_control->HasSelected()) { // actual coin control calculation - CoinControlDialog::updateLabels(model, this); + CoinControlDialog::updateLabels(*m_coin_control, model, this); // show coin control stats ui->labelCoinControlAutomaticallySelected->hide(); diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 36bc2a846b..6961aa7821 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -12,6 +12,7 @@ #include <QString> #include <QTimer> +class CCoinControl; class ClientModel; class PlatformStyle; class SendCoinsEntry; @@ -47,8 +48,8 @@ public: public Q_SLOTS: void clear(); - void reject(); - void accept(); + void reject() override; + void accept() override; SendCoinsEntry *addEntry(); void updateTabsAndLabels(); void setBalance(const interfaces::WalletBalances& balances); @@ -60,6 +61,7 @@ private: Ui::SendCoinsDialog *ui; ClientModel *clientModel; WalletModel *model; + std::unique_ptr<CCoinControl> m_coin_control; std::unique_ptr<WalletModelTransaction> m_current_transaction; bool fNewRecipientAllowed; bool fFeeMinimized; @@ -112,7 +114,7 @@ class SendConfirmationDialog : public QMessageBox public: SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text = "", const QString& detailed_text = "", int secDelay = SEND_CONFIRM_DELAY, const QString& confirmText = "Send", QWidget* parent = nullptr); - int exec(); + int exec() override; private Q_SLOTS: void countDown(); diff --git a/src/qt/sendcoinsrecipient.h b/src/qt/sendcoinsrecipient.h index 12279fab64..6619faf417 100644 --- a/src/qt/sendcoinsrecipient.h +++ b/src/qt/sendcoinsrecipient.h @@ -44,30 +44,21 @@ public: static const int CURRENT_VERSION = 1; int nVersion; - ADD_SERIALIZE_METHODS; + SERIALIZE_METHODS(SendCoinsRecipient, obj) + { + std::string address_str, label_str, message_str, auth_merchant_str; - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - std::string sAddress = address.toStdString(); - std::string sLabel = label.toStdString(); - std::string sMessage = message.toStdString(); - std::string sAuthenticatedMerchant = authenticatedMerchant.toStdString(); + SER_WRITE(obj, address_str = obj.address.toStdString()); + SER_WRITE(obj, label_str = obj.label.toStdString()); + SER_WRITE(obj, message_str = obj.message.toStdString()); + SER_WRITE(obj, auth_merchant_str = obj.authenticatedMerchant.toStdString()); - READWRITE(this->nVersion); - READWRITE(sAddress); - READWRITE(sLabel); - READWRITE(amount); - READWRITE(sMessage); - READWRITE(sPaymentRequest); - READWRITE(sAuthenticatedMerchant); + READWRITE(obj.nVersion, address_str, label_str, obj.amount, message_str, obj.sPaymentRequest, auth_merchant_str); - if (ser_action.ForRead()) - { - address = QString::fromStdString(sAddress); - label = QString::fromStdString(sLabel); - message = QString::fromStdString(sMessage); - authenticatedMerchant = QString::fromStdString(sAuthenticatedMerchant); - } + SER_READ(obj, obj.address = QString::fromStdString(address_str)); + SER_READ(obj, obj.label = QString::fromStdString(label_str)); + SER_READ(obj, obj.message = QString::fromStdString(message_str)); + SER_READ(obj, obj.authenticatedMerchant = QString::fromStdString(auth_merchant_str)); } }; diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 67c2c71e9c..5cfbb67449 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -47,6 +47,8 @@ SignVerifyMessageDialog::SignVerifyMessageDialog(const PlatformStyle *_platformS ui->signatureOut_SM->setFont(GUIUtil::fixedPitchFont()); ui->signatureIn_VM->setFont(GUIUtil::fixedPitchFont()); + + GUIUtil::handleCloseWindowShortcut(this); } SignVerifyMessageDialog::~SignVerifyMessageDialog() diff --git a/src/qt/signverifymessagedialog.h b/src/qt/signverifymessagedialog.h index d2e04cd4fe..d33a2d038d 100644 --- a/src/qt/signverifymessagedialog.h +++ b/src/qt/signverifymessagedialog.h @@ -30,7 +30,7 @@ public: void showTab_VM(bool fShow); protected: - bool eventFilter(QObject *object, QEvent *event); + bool eventFilter(QObject *object, QEvent *event) override; private: Ui::SignVerifyMessageDialog *ui; diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index d8fe4a9238..ced6a299d5 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -127,6 +127,8 @@ SplashScreen::SplashScreen(interfaces::Node& node, Qt::WindowFlags f, const Netw subscribeToCoreSignals(); installEventFilter(this); + + GUIUtil::handleCloseWindowShortcut(this); } SplashScreen::~SplashScreen() diff --git a/src/qt/splashscreen.h b/src/qt/splashscreen.h index f99dd0c701..3158524117 100644 --- a/src/qt/splashscreen.h +++ b/src/qt/splashscreen.h @@ -32,8 +32,8 @@ public: ~SplashScreen(); protected: - void paintEvent(QPaintEvent *event); - void closeEvent(QCloseEvent *event); + void paintEvent(QPaintEvent *event) override; + void closeEvent(QCloseEvent *event) override; public Q_SLOTS: /** Hide the splash screen window and schedule the splash screen object for deletion */ @@ -43,7 +43,7 @@ public Q_SLOTS: void showMessage(const QString &message, int alignment, const QColor &color); protected: - bool eventFilter(QObject * obj, QEvent * ev); + bool eventFilter(QObject * obj, QEvent * ev) override; private: /** Connect core signals to splash screen */ diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 2ba2587852..31feca3253 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -145,7 +145,6 @@ void TestGUI(interfaces::Node& node) wallet->LoadWallet(firstRun); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); - auto locked_chain = wallet->chain().lock(); LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); wallet->SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type), "", "receive"); spk_man->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); diff --git a/src/qt/trafficgraphwidget.cpp b/src/qt/trafficgraphwidget.cpp index 757648f485..6428fc4daf 100644 --- a/src/qt/trafficgraphwidget.cpp +++ b/src/qt/trafficgraphwidget.cpp @@ -7,6 +7,7 @@ #include <qt/clientmodel.h> #include <QPainter> +#include <QPainterPath> #include <QColor> #include <QTimer> diff --git a/src/qt/trafficgraphwidget.h b/src/qt/trafficgraphwidget.h index 7e8bfb2337..2d8c825815 100644 --- a/src/qt/trafficgraphwidget.h +++ b/src/qt/trafficgraphwidget.h @@ -25,7 +25,7 @@ public: int getGraphRangeMins() const; protected: - void paintEvent(QPaintEvent *); + void paintEvent(QPaintEvent *) override; public Q_SLOTS: void updateRates(); diff --git a/src/qt/transactiondescdialog.cpp b/src/qt/transactiondescdialog.cpp index ca72720e00..715e312b19 100644 --- a/src/qt/transactiondescdialog.cpp +++ b/src/qt/transactiondescdialog.cpp @@ -5,6 +5,7 @@ #include <qt/transactiondescdialog.h> #include <qt/forms/ui_transactiondescdialog.h> +#include <qt/guiutil.h> #include <qt/transactiontablemodel.h> #include <QModelIndex> @@ -17,6 +18,8 @@ TransactionDescDialog::TransactionDescDialog(const QModelIndex &idx, QWidget *pa setWindowTitle(tr("Details for %1").arg(idx.data(TransactionTableModel::TxHashRole).toString())); QString desc = idx.data(TransactionTableModel::LongDescriptionRole).toString(); ui->detailText->setHtml(desc); + + GUIUtil::handleCloseWindowShortcut(this); } TransactionDescDialog::~TransactionDescDialog() diff --git a/src/qt/transactionfilterproxy.h b/src/qt/transactionfilterproxy.h index 685f8d3a26..d6bb84f7e6 100644 --- a/src/qt/transactionfilterproxy.h +++ b/src/qt/transactionfilterproxy.h @@ -49,10 +49,10 @@ public: /** Set whether to show conflicted transactions. */ void setShowInactive(bool showInactive); - int rowCount(const QModelIndex &parent = QModelIndex()) const; + int rowCount(const QModelIndex &parent = QModelIndex()) const override; protected: - bool filterAcceptsRow(int source_row, const QModelIndex & source_parent) const; + bool filterAcceptsRow(int source_row, const QModelIndex & source_parent) const override; private: QDateTime dateFrom; diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 18554aef1f..7a15503228 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -664,7 +664,7 @@ QVariant TransactionTableModel::headerData(int section, Qt::Orientation orientat QModelIndex TransactionTableModel::index(int row, int column, const QModelIndex &parent) const { Q_UNUSED(parent); - TransactionRecord *data = priv->index(walletModel->wallet(), walletModel->clientModel().getNumBlocks(), row); + TransactionRecord *data = priv->index(walletModel->wallet(), walletModel->getNumBlocks(), row); if(data) { return createIndex(row, column, data); diff --git a/src/qt/transactiontablemodel.h b/src/qt/transactiontablemodel.h index 7a7d98962b..f06f0ea15f 100644 --- a/src/qt/transactiontablemodel.h +++ b/src/qt/transactiontablemodel.h @@ -76,11 +76,11 @@ public: RawDecorationRole, }; - int rowCount(const QModelIndex &parent) const; - int columnCount(const QModelIndex &parent) const; - QVariant data(const QModelIndex &index, int role) const; - QVariant headerData(int section, Qt::Orientation orientation, int role) const; - QModelIndex index(int row, int column, const QModelIndex & parent = QModelIndex()) const; + int rowCount(const QModelIndex &parent) const override; + int columnCount(const QModelIndex &parent) const override; + QVariant data(const QModelIndex &index, int role) const override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + QModelIndex index(int row, int column, const QModelIndex & parent = QModelIndex()) const override; bool processingQueuedTransactions() const { return fProcessingQueuedTransactions; } private: diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index eca5656077..268e3751b3 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -82,9 +82,9 @@ private: GUIUtil::TableViewLastColumnResizingFixer *columnResizingFixer; - virtual void resizeEvent(QResizeEvent* event); + virtual void resizeEvent(QResizeEvent* event) override; - bool eventFilter(QObject *obj, QEvent *event); + bool eventFilter(QObject *obj, QEvent *event) override; private Q_SLOTS: void contextualMenu(const QPoint &); diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index d16feba1bb..01922cf996 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -10,6 +10,8 @@ #include <qt/forms/ui_helpmessagedialog.h> +#include <qt/guiutil.h> + #include <clientversion.h> #include <init.h> #include <util/system.h> @@ -102,6 +104,8 @@ HelpMessageDialog::HelpMessageDialog(interfaces::Node& node, QWidget *parent, bo ui->scrollArea->setVisible(false); ui->aboutLogo->setVisible(false); } + + GUIUtil::handleCloseWindowShortcut(this); } HelpMessageDialog::~HelpMessageDialog() @@ -141,6 +145,8 @@ ShutdownWindow::ShutdownWindow(QWidget *parent, Qt::WindowFlags f): tr("%1 is shutting down...").arg(PACKAGE_NAME) + "<br /><br />" + tr("Do not shut down the computer until this window disappears."))); setLayout(layout); + + GUIUtil::handleCloseWindowShortcut(this); } QWidget* ShutdownWindow::showShutdownWindow(QMainWindow* window) diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index b6a42d3d9d..425b468f40 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -51,7 +51,7 @@ public: static QWidget* showShutdownWindow(QMainWindow* window); protected: - void closeEvent(QCloseEvent *event); + void closeEvent(QCloseEvent *event) override; }; #endif // BITCOIN_QT_UTILITYDIALOG_H diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 9c1488fb0e..20f2ef5b5f 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -14,6 +14,7 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <util/string.h> +#include <util/translation.h> #include <wallet/wallet.h> #include <algorithm> @@ -244,10 +245,10 @@ void CreateWalletActivity::finish() { destroyProgressDialog(); - if (!m_error_message.empty()) { - QMessageBox::critical(m_parent_widget, tr("Create wallet failed"), QString::fromStdString(m_error_message)); + if (!m_error_message.original.empty()) { + QMessageBox::critical(m_parent_widget, tr("Create wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { - QMessageBox::warning(m_parent_widget, tr("Create wallet warning"), QString::fromStdString(Join(m_warning_message, "\n"))); + QMessageBox::warning(m_parent_widget, tr("Create wallet warning"), QString::fromStdString(Join(m_warning_message, Untranslated("\n")).translated)); } if (m_wallet_model) Q_EMIT created(m_wallet_model); @@ -285,10 +286,10 @@ void OpenWalletActivity::finish() { destroyProgressDialog(); - if (!m_error_message.empty()) { - QMessageBox::critical(m_parent_widget, tr("Open wallet failed"), QString::fromStdString(m_error_message)); + if (!m_error_message.original.empty()) { + QMessageBox::critical(m_parent_widget, tr("Open wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { - QMessageBox::warning(m_parent_widget, tr("Open wallet warning"), QString::fromStdString(Join(m_warning_message, "\n"))); + QMessageBox::warning(m_parent_widget, tr("Open wallet warning"), QString::fromStdString(Join(m_warning_message, Untranslated("\n")).translated)); } if (m_wallet_model) Q_EMIT opened(m_wallet_model); diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index 3808b7d8bf..24dd83adf7 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -8,6 +8,7 @@ #include <qt/sendcoinsrecipient.h> #include <support/allocators/secure.h> #include <sync.h> +#include <util/translation.h> #include <map> #include <memory> @@ -104,8 +105,8 @@ protected: QWidget* const m_parent_widget; QProgressDialog* m_progress_dialog{nullptr}; WalletModel* m_wallet_model{nullptr}; - std::string m_error_message; - std::vector<std::string> m_warning_message; + bilingual_str m_error_message; + std::vector<bilingual_str> m_warning_message; }; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index e7581cd64e..b1e61e03b3 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -24,6 +24,7 @@ #include <psbt.h> #include <ui_interface.h> #include <util/system.h> // for GetBoolArg +#include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/wallet.h> // for CRecipient @@ -38,14 +39,15 @@ WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent) : QObject(parent), m_wallet(std::move(wallet)), - m_client_model(client_model), + m_client_model(&client_model), m_node(client_model.node()), optionsModel(client_model.getOptionsModel()), addressTableModel(nullptr), transactionTableModel(nullptr), recentRequestsTableModel(nullptr), cachedEncryptionStatus(Unencrypted), - cachedNumBlocks(0) + cachedNumBlocks(0), + timer(new QTimer(this)) { fHaveWatchOnly = m_wallet->haveWatchOnly(); addressTableModel = new AddressTableModel(this); @@ -63,11 +65,16 @@ WalletModel::~WalletModel() void WalletModel::startPollBalance() { // This timer will be fired repeatedly to update the balance - QTimer* timer = new QTimer(this); connect(timer, &QTimer::timeout, this, &WalletModel::pollBalanceChanged); timer->start(MODEL_UPDATE_DELAY); } +void WalletModel::setClientModel(ClientModel* client_model) +{ + m_client_model = client_model; + if (!m_client_model) timer->stop(); +} + void WalletModel::updateStatus() { EncryptionStatus newEncryptionStatus = getEncryptionStatus(); @@ -79,24 +86,31 @@ void WalletModel::updateStatus() void WalletModel::pollBalanceChanged() { + // Avoid recomputing wallet balances unless a TransactionChanged or + // BlockTip notification was received. + if (!fForceCheckBalanceChanged && cachedNumBlocks == m_client_model->getNumBlocks()) return; + // Try to get balances and return early if locks can't be acquired. This // avoids the GUI from getting stuck on periodical polls if the core is // holding the locks for a longer time - for example, during a wallet // rescan. interfaces::WalletBalances new_balances; int numBlocks = -1; - if (!m_wallet->tryGetBalances(new_balances, numBlocks, fForceCheckBalanceChanged, cachedNumBlocks)) { + if (!m_wallet->tryGetBalances(new_balances, numBlocks)) { return; } - fForceCheckBalanceChanged = false; + if(fForceCheckBalanceChanged || numBlocks != cachedNumBlocks) + { + fForceCheckBalanceChanged = false; - // Balance and number of transactions might have changed - cachedNumBlocks = numBlocks; + // Balance and number of transactions might have changed + cachedNumBlocks = numBlocks; - checkBalanceChanged(new_balances); - if(transactionTableModel) - transactionTableModel->updateConfirmations(); + checkBalanceChanged(new_balances); + if(transactionTableModel) + transactionTableModel->updateConfirmations(); + } } void WalletModel::checkBalanceChanged(const interfaces::WalletBalances& new_balances) @@ -185,10 +199,10 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact { CAmount nFeeRequired = 0; int nChangePosRet = -1; - std::string strFailReason; + bilingual_str error; auto& newTx = transaction.getWtx(); - newTx = m_wallet->createTransaction(vecSend, coinControl, !wallet().privateKeysDisabled() /* sign */, nChangePosRet, nFeeRequired, strFailReason); + newTx = m_wallet->createTransaction(vecSend, coinControl, !wallet().privateKeysDisabled() /* sign */, nChangePosRet, nFeeRequired, error); transaction.setTransactionFee(nFeeRequired); if (fSubtractFeeFromAmount && newTx) transaction.reassignAmounts(nChangePosRet); @@ -199,8 +213,8 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact { return SendCoinsReturn(AmountWithFeeExceedsBalance); } - Q_EMIT message(tr("Send Coins"), QString::fromStdString(strFailReason), - CClientUIInterface::MSG_ERROR); + Q_EMIT message(tr("Send Coins"), QString::fromStdString(error.translated), + CClientUIInterface::MSG_ERROR); return TransactionCreationFailed; } @@ -303,16 +317,10 @@ WalletModel::EncryptionStatus WalletModel::getEncryptionStatus() const bool WalletModel::setWalletEncrypted(bool encrypted, const SecureString &passphrase) { - if(encrypted) - { - // Encrypt + if (encrypted) { return m_wallet->encryptWallet(passphrase); } - else - { - // Decrypt -- TODO; not supported yet - return false; - } + return false; } bool WalletModel::setWalletLocked(bool locked, const SecureString &passPhrase) @@ -482,14 +490,14 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) { CCoinControl coin_control; coin_control.m_signal_bip125_rbf = true; - std::vector<std::string> errors; + std::vector<bilingual_str> errors; CAmount old_fee; CAmount new_fee; CMutableTransaction mtx; if (!m_wallet->createBumpTransaction(hash, coin_control, errors, old_fee, new_fee, mtx)) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Increasing transaction fee failed") + "<br />(" + - (errors.size() ? QString::fromStdString(errors[0]) : "") +")"); - return false; + (errors.size() ? QString::fromStdString(errors[0].translated) : "") +")"); + return false; } const bool create_psbt = m_wallet->privateKeysDisabled(); @@ -551,8 +559,8 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) // commit the bumped transaction if(!m_wallet->commitBumpTransaction(hash, std::move(mtx), errors, new_hash)) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Could not commit transaction") + "<br />(" + - QString::fromStdString(errors[0])+")"); - return false; + QString::fromStdString(errors[0].translated)+")"); + return false; } return true; } diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 07004b7c6b..23232ec66b 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -144,7 +144,8 @@ public: interfaces::Node& node() const { return m_node; } interfaces::Wallet& wallet() const { return *m_wallet; } - ClientModel& clientModel() const { return m_client_model; } + void setClientModel(ClientModel* client_model); + int getNumBlocks() const { return cachedNumBlocks; } QString getWalletName() const; QString getDisplayName() const; @@ -161,7 +162,7 @@ private: std::unique_ptr<interfaces::Handler> m_handler_show_progress; std::unique_ptr<interfaces::Handler> m_handler_watch_only_changed; std::unique_ptr<interfaces::Handler> m_handler_can_get_addrs_changed; - ClientModel& m_client_model; + ClientModel* m_client_model; interfaces::Node& m_node; bool fHaveWatchOnly; @@ -179,6 +180,7 @@ private: interfaces::WalletBalances m_cached_balances; EncryptionStatus cachedEncryptionStatus; int cachedNumBlocks; + QTimer* timer; void subscribeToCoreSignals(); void unsubscribeFromCoreSignals(); diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 35e36e39be..66fbf978be 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -99,6 +99,7 @@ void WalletView::setClientModel(ClientModel *_clientModel) overviewPage->setClientModel(_clientModel); sendCoinsPage->setClientModel(_clientModel); + if (walletModel) walletModel->setClientModel(_clientModel); } void WalletView::setWalletModel(WalletModel *_walletModel) diff --git a/src/random.cpp b/src/random.cpp index b408b1e13e..9c9a35709a 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -14,16 +14,14 @@ #include <wincrypt.h> #endif #include <logging.h> // for LogPrintf() +#include <randomenv.h> +#include <support/allocators/secure.h> #include <sync.h> // for Mutex #include <util/time.h> // for GetTimeMicros() #include <stdlib.h> #include <thread> -#include <randomenv.h> - -#include <support/allocators/secure.h> - #ifndef WIN32 #include <fcntl.h> #include <sys/time.h> @@ -116,7 +114,10 @@ static uint64_t GetRdRand() noexcept // RdRand may very rarely fail. Invoke it up to 10 times in a loop to reduce this risk. #ifdef __i386__ uint8_t ok; - uint32_t r1, r2; + // Initialize to 0 to silence a compiler warning that r1 or r2 may be used + // uninitialized. Even if rdrand fails (!ok) it will set the output to 0, + // but there is no way that the compiler could know that. + uint32_t r1 = 0, r2 = 0; for (int i = 0; i < 10; ++i) { __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %eax if (ok) break; @@ -128,7 +129,7 @@ static uint64_t GetRdRand() noexcept return (((uint64_t)r2) << 32) | r1; #elif defined(__x86_64__) || defined(__amd64__) uint8_t ok; - uint64_t r1; + uint64_t r1 = 0; // See above why we initialize to 0. for (int i = 0; i < 10; ++i) { __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %rax if (ok) break; @@ -587,16 +588,6 @@ uint64_t GetRand(uint64_t nMax) noexcept return FastRandomContext(g_mock_deterministic_tests).randrange(nMax); } -std::chrono::microseconds GetRandMicros(std::chrono::microseconds duration_max) noexcept -{ - return std::chrono::microseconds{GetRand(duration_max.count())}; -} - -std::chrono::milliseconds GetRandMillis(std::chrono::milliseconds duration_max) noexcept -{ - return std::chrono::milliseconds{GetRand(duration_max.count())}; -} - int GetRandInt(int nMax) noexcept { return GetRand(nMax); diff --git a/src/random.h b/src/random.h index 690125079b..0c6dc24983 100644 --- a/src/random.h +++ b/src/random.h @@ -67,9 +67,21 @@ * Thread-safe. */ void GetRandBytes(unsigned char* buf, int num) noexcept; +/** Generate a uniform random integer in the range [0..range). Precondition: range > 0 */ uint64_t GetRand(uint64_t nMax) noexcept; -std::chrono::microseconds GetRandMicros(std::chrono::microseconds duration_max) noexcept; -std::chrono::milliseconds GetRandMillis(std::chrono::milliseconds duration_max) noexcept; +/** Generate a uniform random duration in the range [0..max). Precondition: max.count() > 0 */ +template <typename D> +D GetRandomDuration(typename std::common_type<D>::type max) noexcept +// Having the compiler infer the template argument from the function argument +// is dangerous, because the desired return value generally has a different +// type than the function argument. So std::common_type is used to force the +// call site to specify the type of the return value. +{ + assert(max.count() > 0); + return D{GetRand(max.count())}; +}; +constexpr auto GetRandMicros = GetRandomDuration<std::chrono::microseconds>; +constexpr auto GetRandMillis = GetRandomDuration<std::chrono::milliseconds>; int GetRandInt(int nMax) noexcept; uint256 GetRandHash() noexcept; diff --git a/src/rest.cpp b/src/rest.cpp index aaaaba4cd0..cde8b472d3 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -18,6 +18,7 @@ #include <sync.h> #include <txmempool.h> #include <util/check.h> +#include <util/ref.h> #include <util/strencodings.h> #include <validation.h> #include <version.h> @@ -49,18 +50,13 @@ struct CCoin { uint32_t nHeight; CTxOut out; - ADD_SERIALIZE_METHODS; - CCoin() : nHeight(0) {} explicit CCoin(Coin&& in) : nHeight(in.nHeight), out(std::move(in.out)) {} - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CCoin, obj) { uint32_t nTxVerDummy = 0; - READWRITE(nTxVerDummy); - READWRITE(nHeight); - READWRITE(out); + READWRITE(nTxVerDummy, obj.nHeight, obj.out); } }; @@ -80,13 +76,14 @@ static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string me * @param[in] req the HTTP request * return pointer to the mempool or nullptr if no mempool found */ -static CTxMemPool* GetMemPool(HTTPRequest* req) +static CTxMemPool* GetMemPool(const util::Ref& context, HTTPRequest* req) { - if (!g_rpc_node || !g_rpc_node->mempool) { + NodeContext* node = context.Has<NodeContext>() ? &context.Get<NodeContext>() : nullptr; + if (!node || !node->mempool) { RESTERR(req, HTTP_NOT_FOUND, "Mempool disabled or instance not found"); return nullptr; } - return g_rpc_node->mempool; + return node->mempool; } static RetFormat ParseDataFormat(std::string& param, const std::string& strReq) @@ -134,7 +131,8 @@ static bool CheckWarmup(HTTPRequest* req) return true; } -static bool rest_headers(HTTPRequest* req, +static bool rest_headers(const util::Ref& context, + HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) @@ -206,7 +204,7 @@ static bool rest_headers(HTTPRequest* req, return true; } default: { - return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex, .json)"); } } } @@ -275,12 +273,12 @@ static bool rest_block(HTTPRequest* req, } } -static bool rest_block_extended(HTTPRequest* req, const std::string& strURIPart) +static bool rest_block_extended(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { return rest_block(req, strURIPart, true); } -static bool rest_block_notxdetails(HTTPRequest* req, const std::string& strURIPart) +static bool rest_block_notxdetails(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { return rest_block(req, strURIPart, false); } @@ -288,7 +286,7 @@ static bool rest_block_notxdetails(HTTPRequest* req, const std::string& strURIPa // A bit of a hack - dependency on a function defined in rpc/blockchain.cpp UniValue getblockchaininfo(const JSONRPCRequest& request); -static bool rest_chaininfo(HTTPRequest* req, const std::string& strURIPart) +static bool rest_chaininfo(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -297,7 +295,7 @@ static bool rest_chaininfo(HTTPRequest* req, const std::string& strURIPart) switch (rf) { case RetFormat::JSON: { - JSONRPCRequest jsonRequest; + JSONRPCRequest jsonRequest(context); jsonRequest.params = UniValue(UniValue::VARR); UniValue chainInfoObject = getblockchaininfo(jsonRequest); std::string strJSON = chainInfoObject.write() + "\n"; @@ -311,11 +309,11 @@ static bool rest_chaininfo(HTTPRequest* req, const std::string& strURIPart) } } -static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) +static bool rest_mempool_info(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; - const CTxMemPool* mempool = GetMemPool(req); + const CTxMemPool* mempool = GetMemPool(context, req); if (!mempool) return false; std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); @@ -335,10 +333,10 @@ static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) } } -static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPart) +static bool rest_mempool_contents(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; - const CTxMemPool* mempool = GetMemPool(req); + const CTxMemPool* mempool = GetMemPool(context, req); if (!mempool) return false; std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); @@ -358,7 +356,7 @@ static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPar } } -static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) +static bool rest_tx(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -414,7 +412,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) } } -static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) +static bool rest_getutxos(const util::Ref& context, HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; @@ -523,7 +521,7 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) }; if (fCheckMemPool) { - const CTxMemPool* mempool = GetMemPool(req); + const CTxMemPool* mempool = GetMemPool(context, req); if (!mempool) return false; // use db+mempool as cache backend in case user likes to query mempool LOCK2(cs_main, mempool->cs); @@ -600,7 +598,7 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) } } -static bool rest_blockhash_by_height(HTTPRequest* req, +static bool rest_blockhash_by_height(const util::Ref& context, HTTPRequest* req, const std::string& str_uri_part) { if (!CheckWarmup(req)) return false; @@ -648,7 +646,7 @@ static bool rest_blockhash_by_height(HTTPRequest* req, static const struct { const char* prefix; - bool (*handler)(HTTPRequest* req, const std::string& strReq); + bool (*handler)(const util::Ref& context, HTTPRequest* req, const std::string& strReq); } uri_prefixes[] = { {"/rest/tx/", rest_tx}, {"/rest/block/notxdetails/", rest_block_notxdetails}, @@ -661,10 +659,12 @@ static const struct { {"/rest/blockhashbyheight/", rest_blockhash_by_height}, }; -void StartREST() +void StartREST(const util::Ref& context) { - for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) - RegisterHTTPHandler(uri_prefixes[i].prefix, false, uri_prefixes[i].handler); + for (const auto& up : uri_prefixes) { + auto handler = [&context, up](HTTPRequest* req, const std::string& prefix) { return up.handler(context, req, prefix); }; + RegisterHTTPHandler(up.prefix, false, handler); + } } void InterruptREST() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 827b83d67e..4eb47d7b15 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -29,6 +29,7 @@ #include <txdb.h> #include <txmempool.h> #include <undo.h> +#include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> #include <validation.h> @@ -51,15 +52,29 @@ struct CUpdatedBlock static Mutex cs_blockchange; static std::condition_variable cond_blockchange; -static CUpdatedBlock latestblock; +static CUpdatedBlock latestblock GUARDED_BY(cs_blockchange); -CTxMemPool& EnsureMemPool() +NodeContext& EnsureNodeContext(const util::Ref& context) { - CHECK_NONFATAL(g_rpc_node); - if (!g_rpc_node->mempool) { + if (!context.Has<NodeContext>()) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Node context not found"); + } + return context.Get<NodeContext>(); +} + +CTxMemPool& EnsureMemPool(const util::Ref& context) +{ + NodeContext& node = EnsureNodeContext(context); + if (!node.mempool) { throw JSONRPCError(RPC_CLIENT_MEMPOOL_DISABLED, "Mempool disabled or instance not found"); } - return *g_rpc_node->mempool; + return *node.mempool; +} + +ChainstateManager& EnsureChainman(const util::Ref& context) +{ + NodeContext& node = EnsureNodeContext(context); + return EnsureChainman(node); } /* Calculate the difficulty for a given block index. @@ -205,10 +220,10 @@ static UniValue getbestblockhash(const JSONRPCRequest& request) return ::ChainActive().Tip()->GetBlockHash().GetHex(); } -void RPCNotifyBlockChange(bool ibd, const CBlockIndex * pindex) +void RPCNotifyBlockChange(const CBlockIndex* pindex) { if(pindex) { - std::lock_guard<std::mutex> lock(cs_blockchange); + LOCK(cs_blockchange); latestblock.hash = pindex->GetBlockHash(); latestblock.height = pindex->nHeight; } @@ -243,9 +258,9 @@ static UniValue waitfornewblock(const JSONRPCRequest& request) WAIT_LOCK(cs_blockchange, lock); block = latestblock; if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&block]{return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); + cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&block]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); else - cond_blockchange.wait(lock, [&block]{return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); + cond_blockchange.wait(lock, [&block]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height != block.height || latestblock.hash != block.hash || !IsRPCRunning(); }); block = latestblock; } UniValue ret(UniValue::VOBJ); @@ -285,9 +300,9 @@ static UniValue waitforblock(const JSONRPCRequest& request) { WAIT_LOCK(cs_blockchange, lock); if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&hash]{return latestblock.hash == hash || !IsRPCRunning();}); + cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&hash]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.hash == hash || !IsRPCRunning();}); else - cond_blockchange.wait(lock, [&hash]{return latestblock.hash == hash || !IsRPCRunning(); }); + cond_blockchange.wait(lock, [&hash]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.hash == hash || !IsRPCRunning(); }); block = latestblock; } @@ -329,9 +344,9 @@ static UniValue waitforblockheight(const JSONRPCRequest& request) { WAIT_LOCK(cs_blockchange, lock); if(timeout) - cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&height]{return latestblock.height >= height || !IsRPCRunning();}); + cond_blockchange.wait_for(lock, std::chrono::milliseconds(timeout), [&height]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height >= height || !IsRPCRunning();}); else - cond_blockchange.wait(lock, [&height]{return latestblock.height >= height || !IsRPCRunning(); }); + cond_blockchange.wait(lock, [&height]() EXCLUSIVE_LOCKS_REQUIRED(cs_blockchange) {return latestblock.height >= height || !IsRPCRunning(); }); block = latestblock; } UniValue ret(UniValue::VOBJ); @@ -399,6 +414,7 @@ static std::vector<RPCResult> MempoolEntryDescription() { return { RPCResult{RPCResult::Type::ARR, "spentby", "unconfirmed transactions spending outputs from this transaction", {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "child transaction id"}}}, RPCResult{RPCResult::Type::BOOL, "bip125-replaceable", "Whether this transaction could be replaced due to BIP125 (replace-by-fee)"}, + RPCResult{RPCResult::Type::BOOL, "unbroadcast", "Whether this transaction is currently unbroadcast (initial broadcast not yet confirmed)"}, };} static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPoolEntry& e) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) @@ -460,6 +476,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool } info.pushKV("bip125-replaceable", rbfStatus); + info.pushKV("unbroadcast", pool.IsUnbroadcastTx(tx.GetHash())); } UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose) @@ -519,7 +536,7 @@ static UniValue getrawmempool(const JSONRPCRequest& request) if (!request.params[0].isNull()) fVerbose = request.params[0].get_bool(); - return MempoolToJSON(EnsureMemPool(), fVerbose); + return MempoolToJSON(EnsureMemPool(request.context), fVerbose); } static UniValue getmempoolancestors(const JSONRPCRequest& request) @@ -549,7 +566,7 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) uint256 hash = ParseHashV(request.params[0], "parameter 1"); - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK(mempool.cs); CTxMemPool::txiter it = mempool.mapTx.find(hash); @@ -612,7 +629,7 @@ static UniValue getmempooldescendants(const JSONRPCRequest& request) uint256 hash = ParseHashV(request.params[0], "parameter 1"); - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK(mempool.cs); CTxMemPool::txiter it = mempool.mapTx.find(hash); @@ -662,7 +679,7 @@ static UniValue getmempoolentry(const JSONRPCRequest& request) uint256 hash = ParseHashV(request.params[0], "parameter 1"); - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK(mempool.cs); CTxMemPool::txiter it = mempool.mapTx.find(hash); @@ -979,7 +996,7 @@ static UniValue gettxoutsetinfo(const JSONRPCRequest& request) ::ChainstateActive().ForceFlushStateToDisk(); CCoinsView* coins_view = WITH_LOCK(cs_main, return &ChainstateActive().CoinsDB()); - if (GetUTXOStats(coins_view, stats)) { + if (GetUTXOStats(coins_view, stats, RpcInterruptionPoint)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); ret.pushKV("transactions", (int64_t)stats.nTransactions); @@ -1045,7 +1062,7 @@ UniValue gettxout(const JSONRPCRequest& request) CCoinsViewCache* coins_view = &::ChainstateActive().CoinsTip(); if (fMempool) { - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK(mempool.cs); CCoinsViewMemPool view(coins_view, mempool); if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { @@ -1389,7 +1406,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) ret.pushKV("maxmempool", (int64_t) maxmempool); ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); - + ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()}); return ret; } @@ -1408,6 +1425,7 @@ static UniValue getmempoolinfo(const JSONRPCRequest& request) {RPCResult::Type::NUM, "maxmempool", "Maximum memory usage for the mempool"}, {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"}, {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, + {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"} }}, RPCExamples{ HelpExampleCli("getmempoolinfo", "") @@ -1415,7 +1433,7 @@ static UniValue getmempoolinfo(const JSONRPCRequest& request) }, }.Check(request); - return MempoolInfoToJSON(EnsureMemPool()); + return MempoolInfoToJSON(EnsureMemPool(request.context)); } static UniValue preciousblock(const JSONRPCRequest& request) @@ -1934,7 +1952,7 @@ static UniValue savemempool(const JSONRPCRequest& request) }, }.Check(request); - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); if (!mempool.IsLoaded()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); @@ -1956,6 +1974,7 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& Coin coin; if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false; if (++count % 8192 == 0) { + RpcInterruptionPoint(); if (should_abort) { // allow to abort the scan via the abort reference return false; @@ -1976,7 +1995,6 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& } /** RAII object to prevent concurrency issue when scanning the txout set */ -static std::mutex g_utxosetscan; static std::atomic<int> g_scan_progress; static std::atomic<bool> g_scan_in_progress; static std::atomic<bool> g_should_abort_scan; @@ -1989,18 +2007,15 @@ public: bool reserve() { CHECK_NONFATAL(!m_could_reserve); - std::lock_guard<std::mutex> lock(g_utxosetscan); - if (g_scan_in_progress) { + if (g_scan_in_progress.exchange(true)) { return false; } - g_scan_in_progress = true; m_could_reserve = true; return true; } ~CoinsViewScanReserver() { if (m_could_reserve) { - std::lock_guard<std::mutex> lock(g_utxosetscan); g_scan_in_progress = false; } } @@ -2242,8 +2257,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) { RPCHelpMan{ "dumptxoutset", - "\nWrite the serialized UTXO set to disk.\n" - "Incidentally flushes the latest coinsdb (leveldb) to disk.\n", + "\nWrite the serialized UTXO set to disk.\n", { {"path", RPCArg::Type::STR, @@ -2300,7 +2314,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) ::ChainstateActive().ForceFlushStateToDisk(); - if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats)) { + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, RpcInterruptionPoint)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } @@ -2318,9 +2332,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) unsigned int iter{0}; while (pcursor->Valid()) { - if (iter % 5000 == 0 && !IsRPCRunning()) { - throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); - } + if (iter % 5000 == 0) RpcInterruptionPoint(); ++iter; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { afile << key; @@ -2386,5 +2398,3 @@ static const CRPCCommand commands[] = for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) t.appendCommand(commands[vcidx].name, &commands[vcidx]); } - -NodeContext* g_rpc_node = nullptr; diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index a02e5fae0e..5c9a43b13e 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -16,8 +16,12 @@ extern RecursiveMutex cs_main; class CBlock; class CBlockIndex; class CTxMemPool; +class ChainstateManager; class UniValue; struct NodeContext; +namespace util { +class Ref; +} // namespace util static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; @@ -30,7 +34,7 @@ static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; double GetDifficulty(const CBlockIndex* blockindex); /** Callback for when block tip changed. */ -void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); +void RPCNotifyBlockChange(const CBlockIndex*); /** Block description to JSON */ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails = false) LOCKS_EXCLUDED(cs_main); @@ -47,11 +51,8 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex /** Used by getblockstats to get feerates at different percentiles by weight */ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector<std::pair<CAmount, int64_t>>& scores, int64_t total_weight); -//! Pointer to node state that needs to be declared as a global to be accessible -//! RPC methods. Due to limitations of the RPC framework, there's currently no -//! direct way to pass in state to RPC methods without globals. -extern NodeContext* g_rpc_node; - -CTxMemPool& EnsureMemPool(); +NodeContext& EnsureNodeContext(const util::Ref& context); +CTxMemPool& EnsureMemPool(const util::Ref& context); +ChainstateManager& EnsureChainman(const util::Ref& context); #endif diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 05d3fd6afb..3612f14bbf 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -101,7 +101,7 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].get_int() : 120, !request.params[1].isNull() ? request.params[1].get_int() : -1); } -static bool GenerateBlock(CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) +static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) { block_hash.SetNull(); @@ -124,14 +124,15 @@ static bool GenerateBlock(CBlock& block, uint64_t& max_tries, unsigned int& extr } std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); - if (!ProcessNewBlock(chainparams, shared_pblock, true, nullptr)) + if (!chainman.ProcessNewBlock(chainparams, shared_pblock, true, nullptr)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); + } block_hash = block.GetHash(); return true; } -static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) +static UniValue generateBlocks(ChainstateManager& chainman, const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { int nHeightEnd = 0; int nHeight = 0; @@ -151,7 +152,7 @@ static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbas CBlock *pblock = &pblocktemplate->block; uint256 block_hash; - if (!GenerateBlock(*pblock, nMaxTries, nExtraNonce, block_hash)) { + if (!GenerateBlock(chainman, *pblock, nMaxTries, nExtraNonce, block_hash)) { break; } @@ -227,9 +228,10 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); } - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); + ChainstateManager& chainman = EnsureChainman(request.context); - return generateBlocks(mempool, coinbase_script, num_blocks, max_tries); + return generateBlocks(chainman, mempool, coinbase_script, num_blocks, max_tries); } static UniValue generatetoaddress(const JSONRPCRequest& request) @@ -265,11 +267,12 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address"); } - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); + ChainstateManager& chainman = EnsureChainman(request.context); CScript coinbase_script = GetScriptForDestination(destination); - return generateBlocks(mempool, coinbase_script, nGenerate, nMaxTries); + return generateBlocks(chainman, mempool, coinbase_script, nGenerate, nMaxTries); } static UniValue generateblock(const JSONRPCRequest& request) @@ -311,7 +314,7 @@ static UniValue generateblock(const JSONRPCRequest& request) coinbase_script = GetScriptForDestination(destination); } - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); std::vector<CTransactionRef> txs; const auto raw_txs_or_txids = request.params[1].get_array(); @@ -370,7 +373,7 @@ static UniValue generateblock(const JSONRPCRequest& request) uint64_t max_tries{1000000}; unsigned int extra_nonce{0}; - if (!GenerateBlock(block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { + if (!GenerateBlock(EnsureChainman(request.context), block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block."); } @@ -403,7 +406,7 @@ static UniValue getmininginfo(const JSONRPCRequest& request) }.Check(request); LOCK(cs_main); - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); UniValue obj(UniValue::VOBJ); obj.pushKV("blocks", (int)::ChainActive().Height()); @@ -449,7 +452,7 @@ static UniValue prioritisetransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Priority is no longer supported, dummy argument to prioritisetransaction must be 0."); } - EnsureMemPool().PrioritiseTransaction(hash, nAmount); + EnsureMemPool(request.context).PrioritiseTransaction(hash, nAmount); return true; } @@ -635,17 +638,18 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) if (strMode != "template") throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode"); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - if (g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) + if (node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); if (::ChainstateActive().IsInitialBlockDownload()) throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); static unsigned int nTransactionsUpdatedLast; - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); if (!lpval.isNull()) { @@ -787,6 +791,8 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) result.pushKV("capabilities", aCaps); UniValue aRules(UniValue::VARR); + aRules.push_back("csv"); + if (!fPreSegWit) aRules.push_back("!segwit"); UniValue vbavailable(UniValue::VOBJ); for (int j = 0; j < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) { Consensus::DeploymentPos pos = Consensus::DeploymentPos(j); @@ -874,7 +880,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) return result; } -class submitblock_StateCatcher : public CValidationInterface +class submitblock_StateCatcher final : public CValidationInterface { public: uint256 hash; @@ -942,17 +948,17 @@ static UniValue submitblock(const JSONRPCRequest& request) } bool new_block; - submitblock_StateCatcher sc(block.GetHash()); - RegisterValidationInterface(&sc); - bool accepted = ProcessNewBlock(Params(), blockptr, /* fForceProcessing */ true, /* fNewBlock */ &new_block); - UnregisterValidationInterface(&sc); + auto sc = std::make_shared<submitblock_StateCatcher>(block.GetHash()); + RegisterSharedValidationInterface(sc); + bool accepted = EnsureChainman(request.context).ProcessNewBlock(Params(), blockptr, /* fForceProcessing */ true, /* fNewBlock */ &new_block); + UnregisterSharedValidationInterface(sc); if (!new_block && accepted) { return "duplicate"; } - if (!sc.found) { + if (!sc->found) { return "inconclusive"; } - return BIP22ValidationResult(sc.state); + return BIP22ValidationResult(sc->state); } static UniValue submitheader(const JSONRPCRequest& request) @@ -983,7 +989,7 @@ static UniValue submitheader(const JSONRPCRequest& request) } BlockValidationState state; - ProcessNewBlockHeaders({h}, state, Params()); + EnsureChainman(request.context).ProcessNewBlockHeaders({h}, state, Params()); if (state.IsValid()) return NullUniValue; if (state.IsError()) { throw JSONRPCError(RPC_VERIFY_ERROR, state.ToString()); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index f3c5fed858..ce98a7c937 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -15,6 +15,7 @@ #include <script/descriptor.h> #include <util/check.h> #include <util/message.h> // For MessageSign(), MessageVerify() +#include <util/ref.h> #include <util/strencodings.h> #include <util/system.h> @@ -366,8 +367,8 @@ static UniValue setmocktime(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VNUM}); int64_t time = request.params[0].get_int64(); SetMockTime(time); - if (g_rpc_node) { - for (const auto& chain_client : g_rpc_node->chain_clients) { + if (request.context.Has<NodeContext>()) { + for (const auto& chain_client : request.context.Get<NodeContext>().chain_clients) { chain_client->setMockTime(time); } } @@ -398,9 +399,10 @@ static UniValue mockscheduler(const JSONRPCRequest& request) } // protect against null pointer dereference - CHECK_NONFATAL(g_rpc_node); - CHECK_NONFATAL(g_rpc_node->scheduler); - g_rpc_node->scheduler->MockForward(std::chrono::seconds(delta_seconds)); + CHECK_NONFATAL(request.context.Has<NodeContext>()); + NodeContext& node = request.context.Get<NodeContext>(); + CHECK_NONFATAL(node.scheduler); + node.scheduler->MockForward(std::chrono::seconds(delta_seconds)); return NullUniValue; } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index d6d15f8b56..e29aa03695 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -42,10 +42,11 @@ static UniValue getconnectioncount(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - return (int)g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL); + return (int)node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL); } static UniValue ping(const JSONRPCRequest& request) @@ -62,11 +63,12 @@ static UniValue ping(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); // Request that each node send a ping during next message processing pass - g_rpc_node->connman->ForEachNode([](CNode* pnode) { + node.connman->ForEachNode([](CNode* pnode) { pnode->fPingQueued = true; }); return NullUniValue; @@ -139,11 +141,12 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); std::vector<CNodeStats> vstats; - g_rpc_node->connman->GetNodeStats(vstats); + node.connman->GetNodeStats(vstats); UniValue ret(UniValue::VARR); @@ -248,7 +251,8 @@ static UniValue addnode(const JSONRPCRequest& request) }, }.ToString()); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); std::string strNode = request.params[0].get_str(); @@ -256,18 +260,18 @@ static UniValue addnode(const JSONRPCRequest& request) if (strCommand == "onetry") { CAddress addr; - g_rpc_node->connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); + node.connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); return NullUniValue; } if (strCommand == "add") { - if(!g_rpc_node->connman->AddNode(strNode)) + if(!node.connman->AddNode(strNode)) throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: Node already added"); } else if(strCommand == "remove") { - if(!g_rpc_node->connman->RemoveAddedNode(strNode)) + if(!node.connman->RemoveAddedNode(strNode)) throw JSONRPCError(RPC_CLIENT_NODE_NOT_ADDED, "Error: Node has not been added."); } @@ -293,7 +297,8 @@ static UniValue disconnectnode(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); bool success; @@ -302,11 +307,11 @@ static UniValue disconnectnode(const JSONRPCRequest& request) if (!address_arg.isNull() && id_arg.isNull()) { /* handle disconnect-by-address */ - success = g_rpc_node->connman->DisconnectNode(address_arg.get_str()); + success = node.connman->DisconnectNode(address_arg.get_str()); } else if (!id_arg.isNull() && (address_arg.isNull() || (address_arg.isStr() && address_arg.get_str().empty()))) { /* handle disconnect-by-id */ NodeId nodeid = (NodeId) id_arg.get_int64(); - success = g_rpc_node->connman->DisconnectNode(nodeid); + success = node.connman->DisconnectNode(nodeid); } else { throw JSONRPCError(RPC_INVALID_PARAMS, "Only one of address and nodeid should be provided."); } @@ -350,10 +355,11 @@ static UniValue getaddednodeinfo(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - std::vector<AddedNodeInfo> vInfo = g_rpc_node->connman->GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = node.connman->GetAddedNodeInfo(); if (!request.params[0].isNull()) { bool found = false; @@ -417,21 +423,22 @@ static UniValue getnettotals(const JSONRPCRequest& request) + HelpExampleRpc("getnettotals", "") }, }.Check(request); - if(!g_rpc_node->connman) + NodeContext& node = EnsureNodeContext(request.context); + if(!node.connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); UniValue obj(UniValue::VOBJ); - obj.pushKV("totalbytesrecv", g_rpc_node->connman->GetTotalBytesRecv()); - obj.pushKV("totalbytessent", g_rpc_node->connman->GetTotalBytesSent()); + obj.pushKV("totalbytesrecv", node.connman->GetTotalBytesRecv()); + obj.pushKV("totalbytessent", node.connman->GetTotalBytesSent()); obj.pushKV("timemillis", GetTimeMillis()); UniValue outboundLimit(UniValue::VOBJ); - outboundLimit.pushKV("timeframe", g_rpc_node->connman->GetMaxOutboundTimeframe()); - outboundLimit.pushKV("target", g_rpc_node->connman->GetMaxOutboundTarget()); - outboundLimit.pushKV("target_reached", g_rpc_node->connman->OutboundTargetReached(false)); - outboundLimit.pushKV("serve_historical_blocks", !g_rpc_node->connman->OutboundTargetReached(true)); - outboundLimit.pushKV("bytes_left_in_cycle", g_rpc_node->connman->GetOutboundTargetBytesLeft()); - outboundLimit.pushKV("time_left_in_cycle", g_rpc_node->connman->GetMaxOutboundTimeLeftInCycle()); + outboundLimit.pushKV("timeframe", node.connman->GetMaxOutboundTimeframe()); + outboundLimit.pushKV("target", node.connman->GetMaxOutboundTarget()); + outboundLimit.pushKV("target_reached", node.connman->OutboundTargetReached(false)); + outboundLimit.pushKV("serve_historical_blocks", !node.connman->OutboundTargetReached(true)); + outboundLimit.pushKV("bytes_left_in_cycle", node.connman->GetOutboundTargetBytesLeft()); + outboundLimit.pushKV("time_left_in_cycle", node.connman->GetMaxOutboundTimeLeftInCycle()); obj.pushKV("uploadtarget", outboundLimit); return obj; } @@ -513,16 +520,17 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) obj.pushKV("version", CLIENT_VERSION); obj.pushKV("subversion", strSubVersion); obj.pushKV("protocolversion",PROTOCOL_VERSION); - if (g_rpc_node->connman) { - ServiceFlags services = g_rpc_node->connman->GetLocalServices(); + NodeContext& node = EnsureNodeContext(request.context); + if (node.connman) { + ServiceFlags services = node.connman->GetLocalServices(); obj.pushKV("localservices", strprintf("%016x", services)); obj.pushKV("localservicesnames", GetServicesNames(services)); } obj.pushKV("localrelay", g_relay_txes); obj.pushKV("timeoffset", GetTimeOffset()); - if (g_rpc_node->connman) { - obj.pushKV("networkactive", g_rpc_node->connman->GetNetworkActive()); - obj.pushKV("connections", (int)g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL)); + if (node.connman) { + obj.pushKV("networkactive", node.connman->GetNetworkActive()); + obj.pushKV("connections", (int)node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL)); } obj.pushKV("networks", GetNetworksInfo()); obj.pushKV("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); @@ -567,7 +575,8 @@ static UniValue setban(const JSONRPCRequest& request) if (request.fHelp || !help.IsValidNumArgs(request.params.size()) || (strCommand != "add" && strCommand != "remove")) { throw std::runtime_error(help.ToString()); } - if (!g_rpc_node->banman) { + NodeContext& node = EnsureNodeContext(request.context); + if (!node.banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } @@ -591,7 +600,7 @@ static UniValue setban(const JSONRPCRequest& request) if (strCommand == "add") { - if (isSubnet ? g_rpc_node->banman->IsBanned(subNet) : g_rpc_node->banman->IsBanned(netAddr)) { + if (isSubnet ? node.banman->IsBanned(subNet) : node.banman->IsBanned(netAddr)) { throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: IP/Subnet already banned"); } @@ -604,20 +613,20 @@ static UniValue setban(const JSONRPCRequest& request) absolute = true; if (isSubnet) { - g_rpc_node->banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); - if (g_rpc_node->connman) { - g_rpc_node->connman->DisconnectNode(subNet); + node.banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); + if (node.connman) { + node.connman->DisconnectNode(subNet); } } else { - g_rpc_node->banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); - if (g_rpc_node->connman) { - g_rpc_node->connman->DisconnectNode(netAddr); + node.banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); + if (node.connman) { + node.connman->DisconnectNode(netAddr); } } } else if(strCommand == "remove") { - if (!( isSubnet ? g_rpc_node->banman->Unban(subNet) : g_rpc_node->banman->Unban(netAddr) )) { + if (!( isSubnet ? node.banman->Unban(subNet) : node.banman->Unban(netAddr) )) { throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously banned."); } } @@ -645,12 +654,13 @@ static UniValue listbanned(const JSONRPCRequest& request) }, }.Check(request); - if(!g_rpc_node->banman) { + NodeContext& node = EnsureNodeContext(request.context); + if(!node.banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } banmap_t banMap; - g_rpc_node->banman->GetBanned(banMap); + node.banman->GetBanned(banMap); UniValue bannedAddresses(UniValue::VARR); for (const auto& entry : banMap) @@ -679,11 +689,12 @@ static UniValue clearbanned(const JSONRPCRequest& request) + HelpExampleRpc("clearbanned", "") }, }.Check(request); - if (!g_rpc_node->banman) { + NodeContext& node = EnsureNodeContext(request.context); + if (!node.banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } - g_rpc_node->banman->ClearBanned(); + node.banman->ClearBanned(); return NullUniValue; } @@ -699,13 +710,14 @@ static UniValue setnetworkactive(const JSONRPCRequest& request) RPCExamples{""}, }.Check(request); - if (!g_rpc_node->connman) { + NodeContext& node = EnsureNodeContext(request.context); + if (!node.connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } - g_rpc_node->connman->SetNetworkActive(request.params[0].get_bool()); + node.connman->SetNetworkActive(request.params[0].get_bool()); - return g_rpc_node->connman->GetNetworkActive(); + return node.connman->GetNetworkActive(); } static UniValue getnodeaddresses(const JSONRPCRequest& request) @@ -732,7 +744,8 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) + HelpExampleRpc("getnodeaddresses", "8") }, }.Check(request); - if (!g_rpc_node->connman) { + NodeContext& node = EnsureNodeContext(request.context); + if (!node.connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -744,7 +757,7 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) } } // returns a shuffled list of CAddress - std::vector<CAddress> vAddr = g_rpc_node->connman->GetAddresses(); + std::vector<CAddress> vAddr = node.connman->GetAddresses(); UniValue ret(UniValue::VARR); int address_return_count = std::min<int>(count, vAddr.size()); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 063ee1697c..e14217c307 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -653,7 +653,7 @@ static UniValue combinerawtransaction(const JSONRPCRequest& request) CCoinsView viewDummy; CCoinsViewCache view(&viewDummy); { - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK(cs_main); LOCK(mempool.cs); CCoinsViewCache &viewChain = ::ChainstateActive().CoinsTip(); @@ -778,7 +778,8 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) for (const CTxIn& txin : mtx.vin) { coins[txin.prevout]; // Create empty map entry keyed by prevout. } - FindCoins(*g_rpc_node, coins); + NodeContext& node = EnsureNodeContext(request.context); + FindCoins(node, coins); // Parse the prevtxs array ParsePrevouts(request.params[2], &keystore, coins); @@ -837,7 +838,8 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) std::string err_string; AssertLockNotHeld(cs_main); - const TransactionError err = BroadcastTransaction(*g_rpc_node, tx, err_string, max_raw_tx_fee, /*relay*/ true, /*wait_callback*/ true); + NodeContext& node = EnsureNodeContext(request.context); + const TransactionError err = BroadcastTransaction(node, tx, err_string, max_raw_tx_fee, /*relay*/ true, /*wait_callback*/ true); if (TransactionError::OK != err) { throw JSONRPCTransactionError(err, err_string); } @@ -904,7 +906,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) DEFAULT_MAX_RAW_TX_FEE_RATE : CFeeRate(AmountFromValue(request.params[1])); - CTxMemPool& mempool = EnsureMemPool(); + CTxMemPool& mempool = EnsureMemPool(request.context); int64_t virtual_size = GetVirtualTransactionSize(*tx); CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); @@ -1555,7 +1557,7 @@ UniValue utxoupdatepsbt(const JSONRPCRequest& request) CCoinsView viewDummy; CCoinsViewCache view(&viewDummy); { - const CTxMemPool& mempool = EnsureMemPool(); + const CTxMemPool& mempool = EnsureMemPool(request.context); LOCK2(cs_main, mempool.cs); CCoinsViewCache &viewChain = ::ChainstateActive().CoinsTip(); CCoinsViewMemPool viewMempool(&viewChain, mempool); diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index 56cac6661e..7fef45f50e 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -130,20 +130,20 @@ void DeleteAuthCookie() } } -std::vector<UniValue> JSONRPCProcessBatchReply(const UniValue &in, size_t num) +std::vector<UniValue> JSONRPCProcessBatchReply(const UniValue& in) { if (!in.isArray()) { throw std::runtime_error("Batch must be an array"); } + const size_t num {in.size()}; std::vector<UniValue> batch(num); - for (size_t i=0; i<in.size(); ++i) { - const UniValue &rec = in[i]; + for (const UniValue& rec : in.getValues()) { if (!rec.isObject()) { - throw std::runtime_error("Batch member must be object"); + throw std::runtime_error("Batch member must be an object"); } size_t id = rec["id"].get_int(); if (id >= num) { - throw std::runtime_error("Batch member id larger than size"); + throw std::runtime_error("Batch member id is larger than batch size"); } batch[id] = rec; } diff --git a/src/rpc/request.h b/src/rpc/request.h index 99eb4f9354..02ec5393a7 100644 --- a/src/rpc/request.h +++ b/src/rpc/request.h @@ -10,6 +10,10 @@ #include <univalue.h> +namespace util { +class Ref; +} // namespace util + UniValue JSONRPCRequestObj(const std::string& strMethod, const UniValue& params, const UniValue& id); UniValue JSONRPCReplyObj(const UniValue& result, const UniValue& error, const UniValue& id); std::string JSONRPCReply(const UniValue& result, const UniValue& error, const UniValue& id); @@ -22,7 +26,7 @@ bool GetAuthCookie(std::string *cookie_out); /** Delete RPC authentication cookie from disk */ void DeleteAuthCookie(); /** Parse JSON-RPC batch reply into a vector */ -std::vector<UniValue> JSONRPCProcessBatchReply(const UniValue &in, size_t num); +std::vector<UniValue> JSONRPCProcessBatchReply(const UniValue& in); class JSONRPCRequest { @@ -34,8 +38,9 @@ public: std::string URI; std::string authUser; std::string peerAddr; + const util::Ref& context; - JSONRPCRequest() : id(NullUniValue), params(NullUniValue), fHelp(false) {} + JSONRPCRequest(const util::Ref& context) : id(NullUniValue), params(NullUniValue), fHelp(false), context(context) {} void parse(const UniValue& valRequest); }; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index e2618c16da..99c649d15a 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -11,9 +11,9 @@ #include <util/strencodings.h> #include <util/system.h> -#include <boost/signals2/signal.hpp> #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/split.hpp> +#include <boost/signals2/signal.hpp> #include <memory> // for unique_ptr #include <unordered_map> @@ -25,7 +25,8 @@ static std::string rpcWarmupStatus GUARDED_BY(cs_rpcWarmup) = "RPC server starte /* Timer-creating functions */ static RPCTimerInterface* timerInterface = nullptr; /* Map of name to timer. */ -static std::map<std::string, std::unique_ptr<RPCTimerBase> > deadlineTimers; +static Mutex g_deadline_timers_mutex; +static std::map<std::string, std::unique_ptr<RPCTimerBase> > deadlineTimers GUARDED_BY(g_deadline_timers_mutex); static bool ExecuteCommand(const CRPCCommand& command, const JSONRPCRequest& request, UniValue& result, bool last_handler); struct RPCCommandExecutionInfo @@ -298,7 +299,7 @@ void InterruptRPC() void StopRPC() { LogPrint(BCLog::RPC, "Stopping RPC\n"); - deadlineTimers.clear(); + WITH_LOCK(g_deadline_timers_mutex, deadlineTimers.clear()); DeleteAuthCookie(); g_rpcSignals.Stopped(); } @@ -308,6 +309,11 @@ bool IsRPCRunning() return g_rpc_running; } +void RpcInterruptionPoint() +{ + if (!IsRPCRunning()) throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); +} + void SetRPCWarmupStatus(const std::string& newStatus) { LOCK(cs_rpcWarmup); @@ -486,6 +492,7 @@ void RPCRunLater(const std::string& name, std::function<void()> func, int64_t nS { if (!timerInterface) throw JSONRPCError(RPC_INTERNAL_ERROR, "No timer handler registered for RPC"); + LOCK(g_deadline_timers_mutex); deadlineTimers.erase(name); LogPrint(BCLog::RPC, "queue run of timer %s in %i seconds (using %s)\n", name, nSeconds, timerInterface->Name()); deadlineTimers.emplace(name, std::unique_ptr<RPCTimerBase>(timerInterface->NewTimer(func, nSeconds*1000))); diff --git a/src/rpc/server.h b/src/rpc/server.h index c91bf1f613..d7a04ff6e8 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -9,10 +9,10 @@ #include <amount.h> #include <rpc/request.h> +#include <functional> #include <map> #include <stdint.h> #include <string> -#include <functional> #include <univalue.h> @@ -29,6 +29,9 @@ namespace RPCServer /** Query whether RPC is running */ bool IsRPCRunning(); +/** Throw JSONRPCError if RPC is not running */ +void RpcInterruptionPoint(); + /** * Set the RPC warmup status. When this is done, all RPC calls will error out * immediately with RPC_IN_WARMUP. diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 5279f40506..ed0175bb10 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -481,7 +481,7 @@ public: return AddChecksum(ret); } - bool ToPrivateString(const SigningProvider& arg, std::string& out) const override final + bool ToPrivateString(const SigningProvider& arg, std::string& out) const final { bool ret = ToStringHelper(&arg, out, true); out = AddChecksum(out); diff --git a/src/script/keyorigin.h b/src/script/keyorigin.h index 467605ce46..a318ff0f9d 100644 --- a/src/script/keyorigin.h +++ b/src/script/keyorigin.h @@ -18,13 +18,7 @@ struct KeyOriginInfo return std::equal(std::begin(a.fingerprint), std::end(a.fingerprint), std::begin(b.fingerprint)) && a.path == b.path; } - ADD_SERIALIZE_METHODS; - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(fingerprint); - READWRITE(path); - } + SERIALIZE_METHODS(KeyOriginInfo, obj) { READWRITE(obj.fingerprint, obj.path); } void clear() { diff --git a/src/script/script.cpp b/src/script/script.cpp index ae0de1d24e..92c6fe7785 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -7,7 +7,9 @@ #include <util/strencodings.h> -const char* GetOpName(opcodetype opcode) +#include <string> + +std::string GetOpName(opcodetype opcode) { switch (opcode) { diff --git a/src/script/script.h b/src/script/script.h index daf4224530..c1f2b66921 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -193,7 +193,7 @@ enum opcodetype // Maximum value that an opcode can be static const unsigned int MAX_OPCODE = OP_NOP10; -const char* GetOpName(opcodetype opcode); +std::string GetOpName(opcodetype opcode); class scriptnum_error : public std::runtime_error { @@ -329,7 +329,7 @@ public: std::vector<unsigned char> result; const bool neg = value < 0; - uint64_t absvalue = neg ? -value : value; + uint64_t absvalue = neg ? ~static_cast<uint64_t>(value) + 1 : static_cast<uint64_t>(value); while(absvalue) { @@ -412,12 +412,7 @@ public: CScript(std::vector<unsigned char>::const_iterator pbegin, std::vector<unsigned char>::const_iterator pend) : CScriptBase(pbegin, pend) { } CScript(const unsigned char* pbegin, const unsigned char* pend) : CScriptBase(pbegin, pend) { } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CScriptBase, *this); - } + SERIALIZE_METHODS(CScript, obj) { READWRITEAS(CScriptBase, obj); } explicit CScript(int64_t b) { operator<<(b); } explicit CScript(opcodetype b) { operator<<(b); } diff --git a/src/script/script_error.cpp b/src/script/script_error.cpp index 57e8fee539..69e14803f1 100644 --- a/src/script/script_error.cpp +++ b/src/script/script_error.cpp @@ -5,7 +5,9 @@ #include <script/script_error.h> -const char* ScriptErrorString(const ScriptError serror) +#include <string> + +std::string ScriptErrorString(const ScriptError serror) { switch (serror) { diff --git a/src/script/script_error.h b/src/script/script_error.h index 400f63ff0f..2978c147e1 100644 --- a/src/script/script_error.h +++ b/src/script/script_error.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_SCRIPT_SCRIPT_ERROR_H #define BITCOIN_SCRIPT_SCRIPT_ERROR_H +#include <string> + typedef enum ScriptError_t { SCRIPT_ERR_OK = 0, @@ -73,6 +75,6 @@ typedef enum ScriptError_t #define SCRIPT_ERR_LAST SCRIPT_ERR_ERROR_COUNT -const char* ScriptErrorString(const ScriptError error); +std::string ScriptErrorString(const ScriptError error); #endif // BITCOIN_SCRIPT_SCRIPT_ERROR_H diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 7d89a336fb..c90c2c24a0 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -9,6 +9,8 @@ #include <pubkey.h> #include <script/script.h> +#include <string> + typedef std::vector<unsigned char> valtype; bool fAcceptDatacarrier = DEFAULT_ACCEPT_DATACARRIER; @@ -25,7 +27,7 @@ WitnessV0ScriptHash::WitnessV0ScriptHash(const CScript& in) CSHA256().Write(in.data(), in.size()).Finalize(begin()); } -const char* GetTxnOutputType(txnouttype t) +std::string GetTxnOutputType(txnouttype t) { switch (t) { @@ -39,7 +41,7 @@ const char* GetTxnOutputType(txnouttype t) case TX_WITNESS_V0_SCRIPTHASH: return "witness_v0_scripthash"; case TX_WITNESS_UNKNOWN: return "witness_unknown"; } - return nullptr; + assert(false); } static bool MatchPayToPubkey(const CScript& script, valtype& pubkey) diff --git a/src/script/standard.h b/src/script/standard.h index 49a45f3eba..2929425670 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -11,6 +11,8 @@ #include <boost/variant.hpp> +#include <string> + static const bool DEFAULT_ACCEPT_DATACARRIER = true; @@ -44,8 +46,7 @@ extern unsigned nMaxDatacarrierBytes; /** * Mandatory script verification flags that all new blocks must comply with for * them to be valid. (but old blocks may not comply with) Currently just P2SH, - * but in the future other flags may be added, such as a soft-fork to enforce - * strict DER encoding. + * but in the future other flags may be added. * * Failing one of these tests may trigger a DoS ban - see CheckInputScripts() for * details. @@ -146,7 +147,7 @@ typedef boost::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, bool IsValidDestination(const CTxDestination& dest); /** Get the name of a txnouttype as a C string, or nullptr if unknown. */ -const char* GetTxnOutputType(txnouttype t); +std::string GetTxnOutputType(txnouttype t); /** * Parse a scriptPubKey and identify script type for standard scripts. If diff --git a/src/serialize.h b/src/serialize.h index fe53eeed31..71c2cfa164 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -43,26 +43,6 @@ static const unsigned int MAX_VECTOR_ALLOCATE = 5000000; struct deserialize_type {}; constexpr deserialize_type deserialize {}; -/** - * Used to bypass the rule against non-const reference to temporary - * where it makes sense with wrappers. - */ -template<typename T> -inline T& REF(const T& val) -{ - return const_cast<T&>(val); -} - -/** - * Used to acquire a non-const pointer "this" to generate bodies - * of const serialization operations from a template - */ -template<typename T> -inline T* NCONST_PTR(const T* val) -{ - return const_cast<T*>(val); -} - //! Safely convert odd char pointer types to standard ones. inline char* CharCast(char* c) { return c; } inline char* CharCast(unsigned char* c) { return (char*)c; } @@ -190,22 +170,8 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; } #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__)) #define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj))) - -/** - * Implement three methods for serializable objects. These are actually wrappers over - * "SerializationOp" template, which implements the body of each class' serialization - * code. Adding "ADD_SERIALIZE_METHODS" in the body of the class causes these wrappers to be - * added as members. - */ -#define ADD_SERIALIZE_METHODS \ - template<typename Stream> \ - void Serialize(Stream& s) const { \ - NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \ - } \ - template<typename Stream> \ - void Unserialize(Stream& s) { \ - SerializationOp(s, CSerActionUnserialize()); \ - } +#define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const<Type>::type& obj) { code; }) +#define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; }) /** * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). @@ -501,7 +467,7 @@ static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T& #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj) #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj) #define COMPACTSIZE(obj) Using<CompactSizeFormatter>(obj) -#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) +#define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj) /** Serialization wrapper class for integers in VarInt format. */ template<VarIntMode Mode> @@ -518,7 +484,16 @@ struct VarIntFormatter } }; -template<int Bytes> +/** Serialization wrapper class for custom integers and enums. + * + * It permits specifying the serialized size (1 to 8 bytes) and endianness. + * + * Use the big endian mode for values that are stored in memory in native + * byte order, but serialized in big endian notation. This is only intended + * to implement serializers that are compatible with existing formats, and + * its use is not recommended for new data structures. + */ +template<int Bytes, bool BigEndian = false> struct CustomUintFormatter { static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range"); @@ -527,52 +502,31 @@ struct CustomUintFormatter template <typename Stream, typename I> void Ser(Stream& s, I v) { if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); - uint64_t raw = htole64(v); - s.write((const char*)&raw, Bytes); + if (BigEndian) { + uint64_t raw = htobe64(v); + s.write(((const char*)&raw) + 8 - Bytes, Bytes); + } else { + uint64_t raw = htole64(v); + s.write((const char*)&raw, Bytes); + } } template <typename Stream, typename I> void Unser(Stream& s, I& v) { - static_assert(std::numeric_limits<I>::max() >= MAX && std::numeric_limits<I>::min() <= 0, "CustomUintFormatter type too small"); + using U = typename std::conditional<std::is_enum<I>::value, std::underlying_type<I>, std::common_type<I>>::type::type; + static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small"); uint64_t raw = 0; - s.read((char*)&raw, Bytes); - v = le64toh(raw); + if (BigEndian) { + s.read(((char*)&raw) + 8 - Bytes, Bytes); + v = static_cast<I>(be64toh(raw)); + } else { + s.read((char*)&raw, Bytes); + v = static_cast<I>(le64toh(raw)); + } } }; -/** Serialization wrapper class for big-endian integers. - * - * Use this wrapper around integer types that are stored in memory in native - * byte order, but serialized in big endian notation. This is only intended - * to implement serializers that are compatible with existing formats, and - * its use is not recommended for new data structures. - * - * Only 16-bit types are supported for now. - */ -template<typename I> -class BigEndian -{ -protected: - I& m_val; -public: - explicit BigEndian(I& val) : m_val(val) - { - static_assert(std::is_unsigned<I>::value, "BigEndian type must be unsigned integer"); - static_assert(sizeof(I) == 2 && std::numeric_limits<I>::min() == 0 && std::numeric_limits<I>::max() == std::numeric_limits<uint16_t>::max(), "Unsupported BigEndian size"); - } - - template<typename Stream> - void Serialize(Stream& s) const - { - ser_writedata16be(s, m_val); - } - - template<typename Stream> - void Unserialize(Stream& s) - { - m_val = ser_readdata16be(s); - } -}; +template<int Bytes> using BigEndianFormatter = CustomUintFormatter<Bytes, true>; /** Formatter for integers in CompactSize format. */ struct CompactSizeFormatter @@ -598,37 +552,26 @@ struct CompactSizeFormatter }; template<size_t Limit> -class LimitedString +struct LimitedStringFormatter { -protected: - std::string& string; -public: - explicit LimitedString(std::string& _string) : string(_string) {} - template<typename Stream> - void Unserialize(Stream& s) + void Unser(Stream& s, std::string& v) { size_t size = ReadCompactSize(s); if (size > Limit) { throw std::ios_base::failure("String length limit exceeded"); } - string.resize(size); - if (size != 0) - s.read((char*)string.data(), size); + v.resize(size); + if (size != 0) s.read((char*)v.data(), size); } template<typename Stream> - void Serialize(Stream& s) const + void Ser(Stream& s, const std::string& v) { - WriteCompactSize(s, string.size()); - if (!string.empty()) - s.write((char*)string.data(), string.size()); + s << v; } }; -template<typename I> -BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); } - /** Formatter to serialize/deserialize vector elements using another formatter * * Example: @@ -1025,7 +968,7 @@ void Unserialize(Stream& is, std::shared_ptr<const T>& p) /** - * Support for ADD_SERIALIZE_METHODS and READWRITE macro + * Support for SERIALIZE_METHODS and READWRITE macro. */ struct CSerActionSerialize { @@ -1124,6 +1067,28 @@ inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&& ::UnserializeMany(s, args...); } +template<typename Stream, typename Type, typename Fn> +inline void SerRead(Stream& s, CSerActionSerialize ser_action, Type&&, Fn&&) +{ +} + +template<typename Stream, typename Type, typename Fn> +inline void SerRead(Stream& s, CSerActionUnserialize ser_action, Type&& obj, Fn&& fn) +{ + fn(s, std::forward<Type>(obj)); +} + +template<typename Stream, typename Type, typename Fn> +inline void SerWrite(Stream& s, CSerActionSerialize ser_action, Type&& obj, Fn&& fn) +{ + fn(s, std::forward<Type>(obj)); +} + +template<typename Stream, typename Type, typename Fn> +inline void SerWrite(Stream& s, CSerActionUnserialize ser_action, Type&&, Fn&&) +{ +} + template<typename I> inline void WriteVarInt(CSizeComputer &s, I n) { diff --git a/src/span.h b/src/span.h index 664ea8d4c6..73b37e35f3 100644 --- a/src/span.h +++ b/src/span.h @@ -25,6 +25,23 @@ public: constexpr Span(C* data, std::ptrdiff_t size) noexcept : m_data(data), m_size(size) {} constexpr Span(C* data, C* end) noexcept : m_data(data), m_size(end - data) {} + /** Implicit conversion of spans between compatible types. + * + * Specifically, if a pointer to an array of type O can be implicitly converted to a pointer to an array of type + * C, then permit implicit conversion of Span<O> to Span<C>. This matches the behavior of the corresponding + * C++20 std::span constructor. + * + * For example this means that a Span<T> can be converted into a Span<const T>. + */ + template <typename O, typename std::enable_if<std::is_convertible<O (*)[], C (*)[]>::value, int>::type = 0> + constexpr Span(const Span<O>& other) noexcept : m_data(other.m_data), m_size(other.m_size) {} + + /** Default copy constructor. */ + constexpr Span(const Span&) noexcept = default; + + /** Default assignment operator. */ + Span& operator=(const Span& other) noexcept = default; + constexpr C* data() const noexcept { return m_data; } constexpr C* begin() const noexcept { return m_data; } constexpr C* end() const noexcept { return m_data + m_size; } @@ -44,6 +61,8 @@ public: friend constexpr bool operator<=(const Span& a, const Span& b) noexcept { return !(b < a); } friend constexpr bool operator>(const Span& a, const Span& b) noexcept { return (b < a); } friend constexpr bool operator>=(const Span& a, const Span& b) noexcept { return !(a < b); } + + template <typename O> friend class Span; }; /** Create a span to a container exposing data() and size(). diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 9b14637b95..f17b539e09 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -253,8 +253,10 @@ void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess) } if (addr) { *lockingSuccess = mlock(addr, len) == 0; -#ifdef MADV_DONTDUMP +#if defined(MADV_DONTDUMP) // Linux madvise(addr, len, MADV_DONTDUMP); +#elif defined(MADV_NOCORE) // FreeBSD + madvise(addr, len, MADV_NOCORE); #endif } return addr; diff --git a/src/sync.cpp b/src/sync.cpp index b86c57e498..9abdedbed4 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -7,15 +7,19 @@ #endif #include <sync.h> -#include <tinyformat.h> #include <logging.h> +#include <tinyformat.h> #include <util/strencodings.h> #include <util/threadnames.h> #include <map> #include <set> #include <system_error> +#include <thread> +#include <unordered_map> +#include <utility> +#include <vector> #ifdef DEBUG_LOCKCONTENTION #if !defined(HAVE_THREAD_LOCAL) @@ -73,35 +77,35 @@ private: int sourceLine; }; -typedef std::vector<std::pair<void*, CLockLocation> > LockStack; -typedef std::map<std::pair<void*, void*>, LockStack> LockOrders; -typedef std::set<std::pair<void*, void*> > InvLockOrders; +using LockStackItem = std::pair<void*, CLockLocation>; +using LockStack = std::vector<LockStackItem>; +using LockStacks = std::unordered_map<std::thread::id, LockStack>; -struct LockData { - // Very ugly hack: as the global constructs and destructors run single - // threaded, we use this boolean to know whether LockData still exists, - // as DeleteLock can get called by global RecursiveMutex destructors - // after LockData disappears. - bool available; - LockData() : available(true) {} - ~LockData() { available = false; } +using LockPair = std::pair<void*, void*>; +using LockOrders = std::map<LockPair, LockStack>; +using InvLockOrders = std::set<LockPair>; +struct LockData { + LockStacks m_lock_stacks; LockOrders lockorders; InvLockOrders invlockorders; std::mutex dd_mutex; }; + LockData& GetLockData() { - static LockData lockdata; - return lockdata; + // This approach guarantees that the object is not destroyed until after its last use. + // The operating system automatically reclaims all the memory in a program's heap when that program exits. + // Since the ~LockData() destructor is never called, the LockData class and all + // its subclasses must have implicitly-defined destructors. + static LockData& lock_data = *new LockData(); + return lock_data; } -static thread_local LockStack g_lockstack; - -static void potential_deadlock_detected(const std::pair<void*, void*>& mismatch, const LockStack& s1, const LockStack& s2) +static void potential_deadlock_detected(const LockPair& mismatch, const LockStack& s1, const LockStack& s2) { LogPrintf("POTENTIAL DEADLOCK DETECTED\n"); LogPrintf("Previous lock order was:\n"); - for (const std::pair<void*, CLockLocation> & i : s2) { + for (const LockStackItem& i : s2) { if (i.first == mismatch.first) { LogPrintf(" (1)"); /* Continued */ } @@ -111,7 +115,7 @@ static void potential_deadlock_detected(const std::pair<void*, void*>& mismatch, LogPrintf(" %s\n", i.second.ToString()); } LogPrintf("Current lock order is:\n"); - for (const std::pair<void*, CLockLocation> & i : s1) { + for (const LockStackItem& i : s1) { if (i.first == mismatch.first) { LogPrintf(" (1)"); /* Continued */ } @@ -132,18 +136,18 @@ static void push_lock(void* c, const CLockLocation& locklocation) LockData& lockdata = GetLockData(); std::lock_guard<std::mutex> lock(lockdata.dd_mutex); - g_lockstack.push_back(std::make_pair(c, locklocation)); - - for (const std::pair<void*, CLockLocation>& i : g_lockstack) { + LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; + lock_stack.emplace_back(c, locklocation); + for (const LockStackItem& i : lock_stack) { if (i.first == c) break; - std::pair<void*, void*> p1 = std::make_pair(i.first, c); + const LockPair p1 = std::make_pair(i.first, c); if (lockdata.lockorders.count(p1)) continue; - lockdata.lockorders.emplace(p1, g_lockstack); + lockdata.lockorders.emplace(p1, lock_stack); - std::pair<void*, void*> p2 = std::make_pair(c, i.first); + const LockPair p2 = std::make_pair(c, i.first); lockdata.invlockorders.insert(p2); if (lockdata.lockorders.count(p2)) potential_deadlock_detected(p1, lockdata.lockorders[p2], lockdata.lockorders[p1]); @@ -152,7 +156,14 @@ static void push_lock(void* c, const CLockLocation& locklocation) static void pop_lock() { - g_lockstack.pop_back(); + LockData& lockdata = GetLockData(); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); + + LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; + lock_stack.pop_back(); + if (lock_stack.empty()) { + lockdata.m_lock_stacks.erase(std::this_thread::get_id()); + } } void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry) @@ -162,11 +173,17 @@ void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) { - if (!g_lockstack.empty()) { - const auto& lastlock = g_lockstack.back(); - if (lastlock.first == cs) { - lockname = lastlock.second.Name(); - return; + { + LockData& lockdata = GetLockData(); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); + + const LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; + if (!lock_stack.empty()) { + const auto& lastlock = lock_stack.back(); + if (lastlock.first == cs) { + lockname = lastlock.second.Name(); + return; + } } } throw std::system_error(EPERM, std::generic_category(), strprintf("%s:%s %s was not most recent critical section locked", file, line, guardname)); @@ -179,49 +196,60 @@ void LeaveCritical() std::string LocksHeld() { + LockData& lockdata = GetLockData(); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); + + const LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; std::string result; - for (const std::pair<void*, CLockLocation>& i : g_lockstack) + for (const LockStackItem& i : lock_stack) result += i.second.ToString() + std::string("\n"); return result; } -void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) +static bool LockHeld(void* mutex) +{ + LockData& lockdata = GetLockData(); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); + + const LockStack& lock_stack = lockdata.m_lock_stacks[std::this_thread::get_id()]; + for (const LockStackItem& i : lock_stack) { + if (i.first == mutex) return true; + } + + return false; +} + +template <typename MutexType> +void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) { - for (const std::pair<void*, CLockLocation>& i : g_lockstack) - if (i.first == cs) - return; + if (LockHeld(cs)) return; tfm::format(std::cerr, "Assertion failed: lock %s not held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); abort(); } +template void AssertLockHeldInternal(const char*, const char*, int, Mutex*); +template void AssertLockHeldInternal(const char*, const char*, int, RecursiveMutex*); void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) { - for (const std::pair<void*, CLockLocation>& i : g_lockstack) { - if (i.first == cs) { - tfm::format(std::cerr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); - abort(); - } - } + if (!LockHeld(cs)) return; + tfm::format(std::cerr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); + abort(); } void DeleteLock(void* cs) { LockData& lockdata = GetLockData(); - if (!lockdata.available) { - // We're already shutting down. - return; - } std::lock_guard<std::mutex> lock(lockdata.dd_mutex); - std::pair<void*, void*> item = std::make_pair(cs, nullptr); + const LockPair item = std::make_pair(cs, nullptr); LockOrders::iterator it = lockdata.lockorders.lower_bound(item); while (it != lockdata.lockorders.end() && it->first.first == cs) { - std::pair<void*, void*> invitem = std::make_pair(it->first.second, it->first.first); + const LockPair invitem = std::make_pair(it->first.second, it->first.first); lockdata.invlockorders.erase(invitem); lockdata.lockorders.erase(it++); } InvLockOrders::iterator invit = lockdata.invlockorders.lower_bound(item); while (invit != lockdata.invlockorders.end() && invit->first == cs) { - std::pair<void*, void*> invinvitem = std::make_pair(invit->second, invit->first); + const LockPair invinvitem = std::make_pair(invit->second, invit->first); lockdata.lockorders.erase(invinvitem); lockdata.invlockorders.erase(invit++); } diff --git a/src/sync.h b/src/sync.h index 0c6f0ef0a7..60e5a87aec 100644 --- a/src/sync.h +++ b/src/sync.h @@ -52,7 +52,8 @@ void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs void LeaveCritical(); void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line); std::string LocksHeld(); -void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) ASSERT_EXCLUSIVE_LOCK(cs); +template <typename MutexType> +void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) ASSERT_EXCLUSIVE_LOCK(cs); void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); void DeleteLock(void* cs); @@ -66,7 +67,8 @@ extern bool g_debug_lockorder_abort; void static inline EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) {} void static inline LeaveCritical() {} void static inline CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) {} -void static inline AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) ASSERT_EXCLUSIVE_LOCK(cs) {} +template <typename MutexType> +void static inline AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) ASSERT_EXCLUSIVE_LOCK(cs) {} void static inline AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} void static inline DeleteLock(void* cs) {} #endif diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 8694891a51..14cf1a4a76 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -132,24 +132,7 @@ public: return base.GetShortID(txhash); } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(header); - READWRITE(nonce); - size_t shorttxids_size = shorttxids.size(); - READWRITE(VARINT(shorttxids_size)); - shorttxids.resize(shorttxids_size); - for (size_t i = 0; i < shorttxids.size(); i++) { - uint32_t lsb = shorttxids[i] & 0xffffffff; - uint16_t msb = (shorttxids[i] >> 32) & 0xffff; - READWRITE(lsb); - READWRITE(msb); - shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb); - } - READWRITE(prefilledtxn); - } + SERIALIZE_METHODS(TestHeaderAndShortIDs, obj) { READWRITE(obj.header, obj.nonce, Using<VectorFormatter<CustomUintFormatter<CBlockHeaderAndShortTxIDs::SHORTTXIDS_LENGTH>>>(obj.shorttxids), obj.prefilledtxn); } }; BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index ff01f730a3..7dff2e6e86 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -43,8 +43,8 @@ static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex BOOST_CHECK(filter_index.LookupFilterHashRange(block_index->nHeight, block_index, filter_hashes)); - BOOST_CHECK_EQUAL(filters.size(), 1); - BOOST_CHECK_EQUAL(filter_hashes.size(), 1); + BOOST_CHECK_EQUAL(filters.size(), 1U); + BOOST_CHECK_EQUAL(filter_hashes.size(), 1U); BOOST_CHECK_EQUAL(filter.GetHash(), expected_filter.GetHash()); BOOST_CHECK_EQUAL(filter_header, expected_filter.ComputeHeader(last_header)); @@ -94,7 +94,7 @@ bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex, CBlockHeader header = block->GetBlockHeader(); BlockValidationState state; - if (!ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { + if (!EnsureChainman(m_node).ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { return false; } } @@ -171,7 +171,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) uint256 chainA_last_header = last_header; for (size_t i = 0; i < 2; i++) { const auto& block = chainA[i]; - BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); } for (size_t i = 0; i < 2; i++) { const auto& block = chainA[i]; @@ -189,7 +189,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) uint256 chainB_last_header = last_header; for (size_t i = 0; i < 3; i++) { const auto& block = chainB[i]; - BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); } for (size_t i = 0; i < 3; i++) { const auto& block = chainB[i]; @@ -220,7 +220,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) // Reorg back to chain A. for (size_t i = 2; i < 4; i++) { const auto& block = chainA[i]; - BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + BOOST_REQUIRE(EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, nullptr)); } // Check that chain A and B blocks can be retrieved. @@ -255,8 +255,9 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) BOOST_CHECK(filter_index.LookupFilterRange(0, tip, filters)); BOOST_CHECK(filter_index.LookupFilterHashRange(0, tip, filter_hashes)); - BOOST_CHECK_EQUAL(filters.size(), tip->nHeight + 1); - BOOST_CHECK_EQUAL(filter_hashes.size(), tip->nHeight + 1); + assert(tip->nHeight >= 0); + BOOST_CHECK_EQUAL(filters.size(), tip->nHeight + 1U); + BOOST_CHECK_EQUAL(filter_hashes.size(), tip->nHeight + 1U); filters.clear(); filter_hashes.clear(); diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index e69503ef35..178c261365 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -42,14 +42,14 @@ BOOST_AUTO_TEST_CASE(gcsfilter_test) BOOST_AUTO_TEST_CASE(gcsfilter_default_constructor) { GCSFilter filter; - BOOST_CHECK_EQUAL(filter.GetN(), 0); - BOOST_CHECK_EQUAL(filter.GetEncoded().size(), 1); + BOOST_CHECK_EQUAL(filter.GetN(), 0U); + BOOST_CHECK_EQUAL(filter.GetEncoded().size(), 1U); const GCSFilter::Params& params = filter.GetParams(); - BOOST_CHECK_EQUAL(params.m_siphash_k0, 0); - BOOST_CHECK_EQUAL(params.m_siphash_k1, 0); + BOOST_CHECK_EQUAL(params.m_siphash_k0, 0U); + BOOST_CHECK_EQUAL(params.m_siphash_k1, 0U); BOOST_CHECK_EQUAL(params.m_P, 0); - BOOST_CHECK_EQUAL(params.m_M, 1); + BOOST_CHECK_EQUAL(params.m_M, 1U); } BOOST_AUTO_TEST_CASE(blockfilter_basic_test) diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index a75d9fc9ec..736c260eeb 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -488,7 +488,7 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) ++nHits; } // Expect about 100 hits - BOOST_CHECK_EQUAL(nHits, 75); + BOOST_CHECK_EQUAL(nHits, 75U); BOOST_CHECK(rb1.contains(data[DATASIZE-1])); rb1.reset(); @@ -516,7 +516,7 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) ++nHits; } // Expect about 5 false positives - BOOST_CHECK_EQUAL(nHits, 6); + BOOST_CHECK_EQUAL(nHits, 6U); // last-1000-entry, 0.01% false positive: CRollingBloomFilter rb2(1000, 0.001); diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 0565982215..35750b2ebc 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <checkqueue.h> +#include <sync.h> #include <test/util/setup_common.h> #include <util/memory.h> #include <util/system.h> @@ -57,14 +58,14 @@ struct FailingCheck { }; struct UniqueCheck { - static std::mutex m; - static std::unordered_multiset<size_t> results; + static Mutex m; + static std::unordered_multiset<size_t> results GUARDED_BY(m); size_t check_id; UniqueCheck(size_t check_id_in) : check_id(check_id_in){}; UniqueCheck() : check_id(0){}; bool operator()() { - std::lock_guard<std::mutex> l(m); + LOCK(m); results.insert(check_id); return true; } @@ -127,7 +128,7 @@ struct FrozenCleanupCheck { std::mutex FrozenCleanupCheck::m{}; std::atomic<uint64_t> FrozenCleanupCheck::nFrozen{0}; std::condition_variable FrozenCleanupCheck::cv{}; -std::mutex UniqueCheck::m; +Mutex UniqueCheck::m; std::unordered_multiset<size_t> UniqueCheck::results; std::atomic<size_t> FakeCheckCheckCompletion::n_calls{0}; std::atomic<size_t> MemoryCheck::fake_allocated_memory{0}; @@ -290,11 +291,15 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) control.Add(vChecks); } } - bool r = true; - BOOST_REQUIRE_EQUAL(UniqueCheck::results.size(), COUNT); - for (size_t i = 0; i < COUNT; ++i) - r = r && UniqueCheck::results.count(i) == 1; - BOOST_REQUIRE(r); + { + LOCK(UniqueCheck::m); + bool r = true; + BOOST_REQUIRE_EQUAL(UniqueCheck::results.size(), COUNT); + for (size_t i = 0; i < COUNT; ++i) { + r = r && UniqueCheck::results.count(i) == 1; + } + BOOST_REQUIRE(r); + } tg.interrupt_all(); tg.join_all(); } diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index df1a119d79..4ddbc8338e 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -70,14 +70,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_ckey_id) CPubKey pubkey = key.GetPubKey(); CScript script = CScript() << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; - BOOST_CHECK_EQUAL(script.size(), 25); + BOOST_CHECK_EQUAL(script.size(), 25U); std::vector<unsigned char> out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script - BOOST_CHECK_EQUAL(out.size(), 21); + BOOST_CHECK_EQUAL(out.size(), 21U); BOOST_CHECK_EQUAL(out[0], 0x00); BOOST_CHECK_EQUAL(memcmp(&out[1], &script[3], 20), 0); // compare the 20 relevant chars of the CKeyId in the script } @@ -87,14 +87,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_cscript_id) // case CScriptID CScript script, redeemScript; script << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; - BOOST_CHECK_EQUAL(script.size(), 23); + BOOST_CHECK_EQUAL(script.size(), 23U); std::vector<unsigned char> out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script - BOOST_CHECK_EQUAL(out.size(), 21); + BOOST_CHECK_EQUAL(out.size(), 21U); BOOST_CHECK_EQUAL(out[0], 0x01); BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 20), 0); // compare the 20 relevant chars of the CScriptId in the script } @@ -105,14 +105,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_compressed_pubkey_id) key.MakeNewKey(true); // case compressed PubKeyID CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // COMPRESSED_PUBLIC_KEY_SIZE (33) - BOOST_CHECK_EQUAL(script.size(), 35); + BOOST_CHECK_EQUAL(script.size(), 35U); std::vector<unsigned char> out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script - BOOST_CHECK_EQUAL(out.size(), 33); + BOOST_CHECK_EQUAL(out.size(), 33U); BOOST_CHECK_EQUAL(memcmp(&out[0], &script[1], 1), 0); BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 32), 0); // compare the 32 chars of the compressed CPubKey } @@ -122,14 +122,14 @@ BOOST_AUTO_TEST_CASE(compress_script_to_uncompressed_pubkey_id) CKey key; key.MakeNewKey(false); // case uncompressed PubKeyID CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // PUBLIC_KEY_SIZE (65) - BOOST_CHECK_EQUAL(script.size(), 67); // 1 char code + 65 char pubkey + OP_CHECKSIG + BOOST_CHECK_EQUAL(script.size(), 67U); // 1 char code + 65 char pubkey + OP_CHECKSIG std::vector<unsigned char> out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); // Check compressed script - BOOST_CHECK_EQUAL(out.size(), 33); + BOOST_CHECK_EQUAL(out.size(), 33U); BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 32), 0); // first 32 chars of CPubKey are copied into out[1:] BOOST_CHECK_EQUAL(out[0], 0x04 | (script[65] & 0x01)); // least significant bit (lsb) of last char of pubkey is mapped into out[0] } diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index c378546e8b..3d802cbeb3 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -331,24 +331,26 @@ struct StringContentsSerializer { } StringContentsSerializer& operator+=(const StringContentsSerializer& s) { return *this += s.str; } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - if (ser_action.ForRead()) { - str.clear(); - char c = 0; - while (true) { - try { - READWRITE(c); - str.push_back(c); - } catch (const std::ios_base::failure&) { - break; - } + template<typename Stream> + void Serialize(Stream& s) const + { + for (size_t i = 0; i < str.size(); i++) { + s << str[i]; + } + } + + template<typename Stream> + void Unserialize(Stream& s) + { + str.clear(); + char c = 0; + while (true) { + try { + s >> c; + str.push_back(c); + } catch (const std::ios_base::failure&) { + break; } - } else { - for (size_t i = 0; i < str.size(); i++) - READWRITE(str[i]); } } }; diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 6314c1a42f..348b170536 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -78,7 +78,7 @@ BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); @@ -148,14 +148,14 @@ static void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerLogicValidat BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { auto connman = MakeUnique<CConnmanTest>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); const Consensus::Params& consensusParams = Params().GetConsensus(); - constexpr int max_outbound_full_relay = 8; + constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; - options.nMaxConnections = 125; + options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; options.m_max_outbound_full_relay = max_outbound_full_relay; - options.nMaxFeeler = 1; + options.nMaxFeeler = MAX_FEELER_CONNECTIONS; connman->Init(options); std::vector<CNode *> vNodes; @@ -221,7 +221,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); banman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); @@ -276,7 +276,7 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); banman->ClearBanned(); gArgs.ForceSetArg("-banscore", "111"); // because 11 is my favorite number @@ -323,7 +323,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) { auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.mempool); + auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); banman->ClearBanned(); int64_t nStartTime = GetTime(); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 121b80ab2d..5f9a78ceb2 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -232,7 +232,7 @@ void DoCheck(const std::string& prv, const std::string& pub, int flags, const st std::vector<CScript> spks_inferred; FlatSigningProvider provider_inferred; BOOST_CHECK(inferred->Expand(0, provider_inferred, spks_inferred, provider_inferred)); - BOOST_CHECK_EQUAL(spks_inferred.size(), 1); + BOOST_CHECK_EQUAL(spks_inferred.size(), 1U); BOOST_CHECK(spks_inferred[0] == spks[n]); BOOST_CHECK_EQUAL(IsSolvable(provider_inferred, spks_inferred[0]), !(flags & UNSOLVABLE)); BOOST_CHECK(provider_inferred.origins == script_provider.origins); diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp index 9bb0b3ef02..be7484cd0b 100644 --- a/src/test/flatfile_tests.cpp +++ b/src/test/flatfile_tests.cpp @@ -93,16 +93,16 @@ BOOST_AUTO_TEST_CASE(flatfile_allocate) bool out_of_space; - BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100); - BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100); + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100U); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100U); BOOST_CHECK(!out_of_space); - BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0); - BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100); + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0U); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100U); BOOST_CHECK(!out_of_space); - BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101); - BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200); + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101U); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200U); BOOST_CHECK(!out_of_space); } @@ -116,11 +116,11 @@ BOOST_AUTO_TEST_CASE(flatfile_flush) // Flush without finalize should not truncate file. seq.Flush(FlatFilePos(0, 1)); - BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100U); // Flush with finalize should truncate file. seq.Flush(FlatFilePos(0, 1), true); - BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1U); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/addrdb.cpp b/src/test/fuzz/addrdb.cpp index f21ff3fac3..524cea83fe 100644 --- a/src/test/fuzz/addrdb.cpp +++ b/src/test/fuzz/addrdb.cpp @@ -3,13 +3,13 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrdb.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cassert> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -30,7 +30,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) })}; break; case 2: { - const Optional<CBanEntry> ban_entry = ConsumeDeserializable<CBanEntry>(fuzzed_data_provider); + const std::optional<CBanEntry> ban_entry = ConsumeDeserializable<CBanEntry>(fuzzed_data_provider); if (ban_entry) { return *ban_entry; } diff --git a/src/test/fuzz/asmap.cpp b/src/test/fuzz/asmap.cpp index 7f3eef79a1..40ca01bd9f 100644 --- a/src/test/fuzz/asmap.cpp +++ b/src/test/fuzz/asmap.cpp @@ -3,26 +3,47 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <netaddress.h> -#include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <cstdint> #include <vector> +//! asmap code that consumes nothing +static const std::vector<bool> IPV6_PREFIX_ASMAP = {}; + +//! asmap code that consumes the 96 prefix bits of ::ffff:0/96 (IPv4-in-IPv6 map) +static const std::vector<bool> IPV4_PREFIX_ASMAP = { + true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 + true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 + true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 + true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 + true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 + true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 + true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 + true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 + true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 + true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00 + true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, // Match 0xFF + true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true // Match 0xFF +}; + void test_one_input(const std::vector<uint8_t>& buffer) { - FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const Network network = fuzzed_data_provider.PickValueInArray({NET_IPV4, NET_IPV6}); - if (fuzzed_data_provider.remaining_bytes() < 16) { - return; - } - CNetAddr net_addr; - net_addr.SetRaw(network, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data()); - std::vector<bool> asmap; - for (const char cur_byte : fuzzed_data_provider.ConsumeRemainingBytes<char>()) { - for (int bit = 0; bit < 8; ++bit) { - asmap.push_back((cur_byte >> bit) & 1); + // Encoding: [7 bits: asmap size] [1 bit: ipv6?] [3-130 bytes: asmap] [4 or 16 bytes: addr] + if (buffer.size() < 1 + 3 + 4) return; + int asmap_size = 3 + (buffer[0] & 127); + bool ipv6 = buffer[0] & 128; + int addr_size = ipv6 ? 16 : 4; + if (buffer.size() < size_t(1 + asmap_size + addr_size)) return; + std::vector<bool> asmap = ipv6 ? IPV6_PREFIX_ASMAP : IPV4_PREFIX_ASMAP; + asmap.reserve(asmap.size() + 8 * asmap_size); + for (int i = 0; i < asmap_size; ++i) { + for (int j = 0; j < 8; ++j) { + asmap.push_back((buffer[1 + i] >> j) & 1); } } + if (!SanityCheckASMap(asmap)) return; + CNetAddr net_addr; + net_addr.SetRaw(ipv6 ? NET_IPV6 : NET_IPV4, buffer.data() + 1 + asmap_size); (void)net_addr.GetMappedAS(asmap); } diff --git a/src/test/fuzz/asmap_direct.cpp b/src/test/fuzz/asmap_direct.cpp new file mode 100644 index 0000000000..2d21eff9d6 --- /dev/null +++ b/src/test/fuzz/asmap_direct.cpp @@ -0,0 +1,48 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/fuzz.h> +#include <util/asmap.h> + +#include <cstdint> +#include <optional> +#include <vector> + +#include <assert.h> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + // Encoding: [asmap using 1 bit / byte] 0xFF [addr using 1 bit / byte] + std::optional<size_t> sep_pos_opt; + for (size_t pos = 0; pos < buffer.size(); ++pos) { + uint8_t x = buffer[pos]; + if ((x & 0xFE) == 0) continue; + if (x == 0xFF) { + if (sep_pos_opt) return; + sep_pos_opt = pos; + } else { + return; + } + } + if (!sep_pos_opt) return; // Needs exactly 1 separator + const size_t sep_pos{sep_pos_opt.value()}; + if (buffer.size() - sep_pos - 1 > 128) return; // At most 128 bits in IP address + + // Checks on asmap + std::vector<bool> asmap(buffer.begin(), buffer.begin() + sep_pos); + if (SanityCheckASMap(asmap, buffer.size() - 1 - sep_pos)) { + // Verify that for valid asmaps, no prefix (except up to 7 zero padding bits) is valid. + std::vector<bool> asmap_prefix = asmap; + while (!asmap_prefix.empty() && asmap_prefix.size() + 7 > asmap.size() && asmap_prefix.back() == false) { + asmap_prefix.pop_back(); + } + while (!asmap_prefix.empty()) { + asmap_prefix.pop_back(); + assert(!SanityCheckASMap(asmap_prefix, buffer.size() - 1 - sep_pos)); + } + // No address input should trigger assertions in interpreter + std::vector<bool> addr(buffer.begin() + sep_pos + 1, buffer.end()); + (void)Interpret(asmap, addr); + } +} diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index f30fa03e0b..91bd34a251 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -38,12 +38,17 @@ void test_one_input(const std::vector<uint8_t>& buffer) const Consensus::Params& consensus_params = Params().GetConsensus(); BlockValidationState validation_state_pow_and_merkle; const bool valid_incl_pow_and_merkle = CheckBlock(block, validation_state_pow_and_merkle, consensus_params, /* fCheckPOW= */ true, /* fCheckMerkleRoot= */ true); + assert(validation_state_pow_and_merkle.IsValid() || validation_state_pow_and_merkle.IsInvalid() || validation_state_pow_and_merkle.IsError()); + (void)validation_state_pow_and_merkle.Error(""); BlockValidationState validation_state_pow; const bool valid_incl_pow = CheckBlock(block, validation_state_pow, consensus_params, /* fCheckPOW= */ true, /* fCheckMerkleRoot= */ false); + assert(validation_state_pow.IsValid() || validation_state_pow.IsInvalid() || validation_state_pow.IsError()); BlockValidationState validation_state_merkle; const bool valid_incl_merkle = CheckBlock(block, validation_state_merkle, consensus_params, /* fCheckPOW= */ false, /* fCheckMerkleRoot= */ true); + assert(validation_state_merkle.IsValid() || validation_state_merkle.IsInvalid() || validation_state_merkle.IsError()); BlockValidationState validation_state_none; const bool valid_incl_none = CheckBlock(block, validation_state_none, consensus_params, /* fCheckPOW= */ false, /* fCheckMerkleRoot= */ false); + assert(validation_state_none.IsValid() || validation_state_none.IsInvalid() || validation_state_none.IsError()); if (valid_incl_pow_and_merkle) { assert(valid_incl_pow && valid_incl_merkle && valid_incl_none); } else if (valid_incl_merkle || valid_incl_pow) { diff --git a/src/test/fuzz/block_header.cpp b/src/test/fuzz/block_header.cpp index 92dcccc0e1..09c2b4a951 100644 --- a/src/test/fuzz/block_header.cpp +++ b/src/test/fuzz/block_header.cpp @@ -2,7 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <optional.h> #include <primitives/block.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> @@ -11,13 +10,14 @@ #include <cassert> #include <cstdint> +#include <optional> #include <string> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const Optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); + const std::optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); if (!block_header) { return; } @@ -38,4 +38,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) block.SetNull(); assert(block.GetBlockHeader().GetHash() == mut_block_header.GetHash()); } + { + std::optional<CBlockLocator> block_locator = ConsumeDeserializable<CBlockLocator>(fuzzed_data_provider); + if (block_locator) { + (void)block_locator->IsNull(); + block_locator->SetNull(); + assert(block_locator->IsNull()); + } + } } diff --git a/src/test/fuzz/blockfilter.cpp b/src/test/fuzz/blockfilter.cpp index be9320dcbf..7232325a20 100644 --- a/src/test/fuzz/blockfilter.cpp +++ b/src/test/fuzz/blockfilter.cpp @@ -3,19 +3,19 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <blockfilter.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cstdint> +#include <optional> #include <string> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const Optional<BlockFilter> block_filter = ConsumeDeserializable<BlockFilter>(fuzzed_data_provider); + const std::optional<BlockFilter> block_filter = ConsumeDeserializable<BlockFilter>(fuzzed_data_provider); if (!block_filter) { return; } diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index 50036ce5bd..d955c71bc9 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bloom.h> -#include <optional.h> #include <primitives/transaction.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> @@ -12,6 +11,7 @@ #include <cassert> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -25,7 +25,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; while (fuzzed_data_provider.remaining_bytes() > 0) { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 4)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 3)) { case 0: { const std::vector<unsigned char> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); (void)bloom_filter.contains(b); @@ -35,7 +35,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } case 1: { - const Optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!out_point) { break; } @@ -46,7 +46,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } case 2: { - const Optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); + const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (!u256) { break; } @@ -57,7 +57,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } case 3: { - const Optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mut_tx) { break; } @@ -65,9 +65,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)bloom_filter.IsRelevantAndUpdate(tx); break; } - case 4: - bloom_filter.UpdateEmptyFull(); - break; } (void)bloom_filter.IsWithinSizeConstraints(); } diff --git a/src/test/fuzz/chain.cpp b/src/test/fuzz/chain.cpp index b322516cc7..47c71850ce 100644 --- a/src/test/fuzz/chain.cpp +++ b/src/test/fuzz/chain.cpp @@ -3,18 +3,18 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cstdint> +#include <optional> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - Optional<CDiskBlockIndex> disk_block_index = ConsumeDeserializable<CDiskBlockIndex>(fuzzed_data_provider); + std::optional<CDiskBlockIndex> disk_block_index = ConsumeDeserializable<CDiskBlockIndex>(fuzzed_data_provider); if (!disk_block_index) { return; } diff --git a/src/test/fuzz/checkqueue.cpp b/src/test/fuzz/checkqueue.cpp index 2ed097b827..c69043bb6b 100644 --- a/src/test/fuzz/checkqueue.cpp +++ b/src/test/fuzz/checkqueue.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <checkqueue.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp new file mode 100644 index 0000000000..52dd62a145 --- /dev/null +++ b/src/test/fuzz/coins_view.cpp @@ -0,0 +1,294 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <amount.h> +#include <chainparams.h> +#include <chainparamsbase.h> +#include <coins.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <key.h> +#include <node/coinstats.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <pubkey.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <validation.h> + +#include <cstdint> +#include <limits> +#include <optional> +#include <string> +#include <vector> + +namespace { +const Coin EMPTY_COIN{}; + +bool operator==(const Coin& a, const Coin& b) +{ + if (a.IsSpent() && b.IsSpent()) return true; + return a.fCoinBase == b.fCoinBase && a.nHeight == b.nHeight && a.out == b.out; +} +} // namespace + +void initialize() +{ + static const ECCVerifyHandle ecc_verify_handle; + ECC_Start(); + SelectParams(CBaseChainParams::REGTEST); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CCoinsView backend_coins_view; + CCoinsViewCache coins_view_cache{&backend_coins_view}; + COutPoint random_out_point; + Coin random_coin; + CMutableTransaction random_mutable_transaction; + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 9)) { + case 0: { + if (random_coin.IsSpent()) { + break; + } + Coin coin = random_coin; + bool expected_code_path = false; + const bool possible_overwrite = fuzzed_data_provider.ConsumeBool(); + try { + coins_view_cache.AddCoin(random_out_point, std::move(coin), possible_overwrite); + expected_code_path = true; + } catch (const std::logic_error& e) { + if (e.what() == std::string{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"}) { + assert(!possible_overwrite); + expected_code_path = true; + } + } + assert(expected_code_path); + break; + } + case 1: { + (void)coins_view_cache.Flush(); + break; + } + case 2: { + coins_view_cache.SetBestBlock(ConsumeUInt256(fuzzed_data_provider)); + break; + } + case 3: { + Coin move_to; + (void)coins_view_cache.SpendCoin(random_out_point, fuzzed_data_provider.ConsumeBool() ? &move_to : nullptr); + break; + } + case 4: { + coins_view_cache.Uncache(random_out_point); + break; + } + case 5: { + if (fuzzed_data_provider.ConsumeBool()) { + backend_coins_view = CCoinsView{}; + } + coins_view_cache.SetBackend(backend_coins_view); + break; + } + case 6: { + const std::optional<COutPoint> opt_out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (!opt_out_point) { + break; + } + random_out_point = *opt_out_point; + break; + } + case 7: { + const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); + if (!opt_coin) { + break; + } + random_coin = *opt_coin; + break; + } + case 8: { + const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!opt_mutable_transaction) { + break; + } + random_mutable_transaction = *opt_mutable_transaction; + break; + } + case 9: { + CCoinsMap coins_map; + while (fuzzed_data_provider.ConsumeBool()) { + CCoinsCacheEntry coins_cache_entry; + coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>(); + if (fuzzed_data_provider.ConsumeBool()) { + coins_cache_entry.coin = random_coin; + } else { + const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); + if (!opt_coin) { + break; + } + coins_cache_entry.coin = *opt_coin; + } + coins_map.emplace(random_out_point, std::move(coins_cache_entry)); + } + bool expected_code_path = false; + try { + coins_view_cache.BatchWrite(coins_map, fuzzed_data_provider.ConsumeBool() ? ConsumeUInt256(fuzzed_data_provider) : coins_view_cache.GetBestBlock()); + expected_code_path = true; + } catch (const std::logic_error& e) { + if (e.what() == std::string{"FRESH flag misapplied to coin that exists in parent cache"}) { + expected_code_path = true; + } + } + assert(expected_code_path); + break; + } + } + } + + { + const Coin& coin_using_access_coin = coins_view_cache.AccessCoin(random_out_point); + const bool exists_using_access_coin = !(coin_using_access_coin == EMPTY_COIN); + const bool exists_using_have_coin = coins_view_cache.HaveCoin(random_out_point); + const bool exists_using_have_coin_in_cache = coins_view_cache.HaveCoinInCache(random_out_point); + Coin coin_using_get_coin; + const bool exists_using_get_coin = coins_view_cache.GetCoin(random_out_point, coin_using_get_coin); + if (exists_using_get_coin) { + assert(coin_using_get_coin == coin_using_access_coin); + } + assert((exists_using_access_coin && exists_using_have_coin_in_cache && exists_using_have_coin && exists_using_get_coin) || + (!exists_using_access_coin && !exists_using_have_coin_in_cache && !exists_using_have_coin && !exists_using_get_coin)); + const bool exists_using_have_coin_in_backend = backend_coins_view.HaveCoin(random_out_point); + if (exists_using_have_coin_in_backend) { + assert(exists_using_have_coin); + } + Coin coin_using_backend_get_coin; + if (backend_coins_view.GetCoin(random_out_point, coin_using_backend_get_coin)) { + assert(exists_using_have_coin_in_backend); + assert(coin_using_get_coin == coin_using_backend_get_coin); + } else { + assert(!exists_using_have_coin_in_backend); + } + } + + { + bool expected_code_path = false; + try { + (void)coins_view_cache.Cursor(); + } catch (const std::logic_error&) { + expected_code_path = true; + } + assert(expected_code_path); + (void)coins_view_cache.DynamicMemoryUsage(); + (void)coins_view_cache.EstimateSize(); + (void)coins_view_cache.GetBestBlock(); + (void)coins_view_cache.GetCacheSize(); + (void)coins_view_cache.GetHeadBlocks(); + (void)coins_view_cache.HaveInputs(CTransaction{random_mutable_transaction}); + } + + { + const CCoinsViewCursor* coins_view_cursor = backend_coins_view.Cursor(); + assert(coins_view_cursor == nullptr); + (void)backend_coins_view.EstimateSize(); + (void)backend_coins_view.GetBestBlock(); + (void)backend_coins_view.GetHeadBlocks(); + } + + if (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 6)) { + case 0: { + const CTransaction transaction{random_mutable_transaction}; + bool is_spent = false; + for (const CTxOut& tx_out : transaction.vout) { + if (Coin{tx_out, 0, transaction.IsCoinBase()}.IsSpent()) { + is_spent = true; + } + } + if (is_spent) { + // Avoid: + // coins.cpp:69: void CCoinsViewCache::AddCoin(const COutPoint &, Coin &&, bool): Assertion `!coin.IsSpent()' failed. + break; + } + bool expected_code_path = false; + const int height = fuzzed_data_provider.ConsumeIntegral<int>(); + const bool possible_overwrite = fuzzed_data_provider.ConsumeBool(); + try { + AddCoins(coins_view_cache, transaction, height, possible_overwrite); + expected_code_path = true; + } catch (const std::logic_error& e) { + if (e.what() == std::string{"Attempted to overwrite an unspent coin (when possible_overwrite is false)"}) { + assert(!possible_overwrite); + expected_code_path = true; + } + } + assert(expected_code_path); + break; + } + case 1: { + (void)AreInputsStandard(CTransaction{random_mutable_transaction}, coins_view_cache); + break; + } + case 2: { + TxValidationState state; + CAmount tx_fee_out; + const CTransaction transaction{random_mutable_transaction}; + if (ContainsSpentInput(transaction, coins_view_cache)) { + // Avoid: + // consensus/tx_verify.cpp:171: bool Consensus::CheckTxInputs(const CTransaction &, TxValidationState &, const CCoinsViewCache &, int, CAmount &): Assertion `!coin.IsSpent()' failed. + break; + } + try { + (void)Consensus::CheckTxInputs(transaction, state, coins_view_cache, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, std::numeric_limits<int>::max()), tx_fee_out); + assert(MoneyRange(tx_fee_out)); + } catch (const std::runtime_error&) { + } + break; + } + case 3: { + const CTransaction transaction{random_mutable_transaction}; + if (ContainsSpentInput(transaction, coins_view_cache)) { + // Avoid: + // consensus/tx_verify.cpp:130: unsigned int GetP2SHSigOpCount(const CTransaction &, const CCoinsViewCache &): Assertion `!coin.IsSpent()' failed. + break; + } + (void)GetP2SHSigOpCount(transaction, coins_view_cache); + break; + } + case 4: { + const CTransaction transaction{random_mutable_transaction}; + if (ContainsSpentInput(transaction, coins_view_cache)) { + // Avoid: + // consensus/tx_verify.cpp:130: unsigned int GetP2SHSigOpCount(const CTransaction &, const CCoinsViewCache &): Assertion `!coin.IsSpent()' failed. + break; + } + const int flags = fuzzed_data_provider.ConsumeIntegral<int>(); + if (!transaction.vin.empty() && (flags & SCRIPT_VERIFY_WITNESS) != 0 && (flags & SCRIPT_VERIFY_P2SH) == 0) { + // Avoid: + // script/interpreter.cpp:1705: size_t CountWitnessSigOps(const CScript &, const CScript &, const CScriptWitness *, unsigned int): Assertion `(flags & SCRIPT_VERIFY_P2SH) != 0' failed. + break; + } + (void)GetTransactionSigOpCost(transaction, coins_view_cache, flags); + break; + } + case 5: { + CCoinsStats stats; + bool expected_code_path = false; + try { + (void)GetUTXOStats(&coins_view_cache, stats); + } catch (const std::logic_error&) { + expected_code_path = true; + } + assert(expected_code_path); + break; + } + case 6: { + (void)IsWitnessStandard(CTransaction{random_mutable_transaction}, coins_view_cache); + break; + } + } + } +} diff --git a/src/test/fuzz/cuckoocache.cpp b/src/test/fuzz/cuckoocache.cpp index f674efe1b1..5b45aa79d8 100644 --- a/src/test/fuzz/cuckoocache.cpp +++ b/src/test/fuzz/cuckoocache.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <cuckoocache.h> -#include <optional.h> #include <script/sigcache.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index 090994263e..ce8700befa 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -3,11 +3,11 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <amount.h> -#include <optional.h> #include <policy/fees.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/fees.h> #include <cstdint> #include <string> @@ -23,4 +23,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) const CAmount rounded_fee = fee_filter_rounder.round(current_minimum_fee); assert(MoneyRange(rounded_fee)); } + const FeeReason fee_reason = fuzzed_data_provider.PickValueInArray({FeeReason::NONE, FeeReason::HALF_ESTIMATE, FeeReason::FULL_ESTIMATE, FeeReason::DOUBLE_ESTIMATE, FeeReason::CONSERVATIVE, FeeReason::MEMPOOL_MIN, FeeReason::PAYTXFEE, FeeReason::FALLBACK, FeeReason::REQUIRED}); + (void)StringForFeeReason(fee_reason); } diff --git a/src/test/fuzz/flatfile.cpp b/src/test/fuzz/flatfile.cpp index a55de77df7..95dabb8bab 100644 --- a/src/test/fuzz/flatfile.cpp +++ b/src/test/fuzz/flatfile.cpp @@ -3,24 +3,24 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <flatfile.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cassert> #include <cstdint> +#include <optional> #include <string> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - Optional<FlatFilePos> flat_file_pos = ConsumeDeserializable<FlatFilePos>(fuzzed_data_provider); + std::optional<FlatFilePos> flat_file_pos = ConsumeDeserializable<FlatFilePos>(fuzzed_data_provider); if (!flat_file_pos) { return; } - Optional<FlatFilePos> another_flat_file_pos = ConsumeDeserializable<FlatFilePos>(fuzzed_data_provider); + std::optional<FlatFilePos> another_flat_file_pos = ConsumeDeserializable<FlatFilePos>(fuzzed_data_provider); if (another_flat_file_pos) { assert((*flat_file_pos == *another_flat_file_pos) != (*flat_file_pos != *another_flat_file_pos)); } diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 6e2188fe86..82e1d55c0b 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -19,8 +19,6 @@ static bool read_stdin(std::vector<uint8_t>& data) ssize_t length = 0; while ((length = read(STDIN_FILENO, buffer, 1024)) > 0) { data.insert(data.end(), buffer, buffer + length); - - if (data.size() > (1 << 20)) return false; } return length == 0; } diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp index 3e20416116..a9f450b0c4 100644 --- a/src/test/fuzz/golomb_rice.cpp +++ b/src/test/fuzz/golomb_rice.cpp @@ -5,8 +5,8 @@ #include <blockfilter.h> #include <serialize.h> #include <streams.h> -#include <test/fuzz/fuzz.h> #include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <util/bytevectorhash.h> #include <util/golombrice.h> diff --git a/src/test/fuzz/hex.cpp b/src/test/fuzz/hex.cpp index 5fed17c17c..6a8699fd0f 100644 --- a/src/test/fuzz/hex.cpp +++ b/src/test/fuzz/hex.cpp @@ -16,7 +16,8 @@ #include <string> #include <vector> -void initialize() { +void initialize() +{ static const ECCVerifyHandle verify_handle; } diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 9dbf0fcc90..35d6804d4f 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -24,8 +24,8 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> -#include <time.h> #include <uint256.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/strencodings.h> #include <util/string.h> @@ -35,6 +35,7 @@ #include <cassert> #include <chrono> +#include <ctime> #include <limits> #include <set> #include <vector> @@ -147,11 +148,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) const CScriptNum script_num{i64}; (void)script_num.getint(); - // Avoid negation failure: - // script/script.h:332:35: runtime error: negation of -9223372036854775808 cannot be represented in type 'int64_t' (aka 'long'); cast to an unsigned type to negate this value to itself - if (script_num != CScriptNum{std::numeric_limits<int64_t>::min()}) { - (void)script_num.getvch(); - } + (void)script_num.getvch(); const arith_uint256 au256 = UintToArith256(u256); assert(ArithToUint256(au256) == u256); @@ -287,8 +284,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) try { const uint64_t deserialized_u64 = ReadCompactSize(stream); assert(u64 == deserialized_u64 && stream.empty()); + } catch (const std::ios_base::failure&) { } - catch (const std::ios_base::failure&) { - } + } + + try { + CHECK_NONFATAL(b); + } catch (const NonFatalCheckError&) { } } diff --git a/src/test/fuzz/kitchen_sink.cpp b/src/test/fuzz/kitchen_sink.cpp new file mode 100644 index 0000000000..af6dc71322 --- /dev/null +++ b/src/test/fuzz/kitchen_sink.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <rpc/util.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/error.h> + +#include <cstdint> +#include <vector> + +// The fuzzing kitchen sink: Fuzzing harness for functions that need to be +// fuzzed but a.) don't belong in any existing fuzzing harness file, and +// b.) are not important enough to warrant their own fuzzing harness file. +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const TransactionError transaction_error = fuzzed_data_provider.PickValueInArray<TransactionError>({TransactionError::OK, TransactionError::MISSING_INPUTS, TransactionError::ALREADY_IN_CHAIN, TransactionError::P2P_DISABLED, TransactionError::MEMPOOL_REJECTED, TransactionError::MEMPOOL_ERROR, TransactionError::INVALID_PSBT, TransactionError::PSBT_MISMATCH, TransactionError::SIGHASH_MISMATCH, TransactionError::MAX_FEE_EXCEEDED}); + (void)JSONRPCTransactionError(transaction_error); + (void)RPCErrorFromTransactionError(transaction_error); + (void)TransactionErrorString(transaction_error); +} diff --git a/src/test/fuzz/merkleblock.cpp b/src/test/fuzz/merkleblock.cpp index eb8fa1d421..c44e334272 100644 --- a/src/test/fuzz/merkleblock.cpp +++ b/src/test/fuzz/merkleblock.cpp @@ -3,20 +3,20 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <merkleblock.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <uint256.h> #include <cstdint> +#include <optional> #include <string> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - Optional<CPartialMerkleTree> partial_merkle_tree = ConsumeDeserializable<CPartialMerkleTree>(fuzzed_data_provider); + std::optional<CPartialMerkleTree> partial_merkle_tree = ConsumeDeserializable<CPartialMerkleTree>(fuzzed_data_provider); if (!partial_merkle_tree) { return; } diff --git a/src/test/fuzz/message.cpp b/src/test/fuzz/message.cpp new file mode 100644 index 0000000000..fa0322a391 --- /dev/null +++ b/src/test/fuzz/message.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <key_io.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/message.h> +#include <util/strencodings.h> + +#include <cassert> +#include <cstdint> +#include <iostream> +#include <string> +#include <vector> + +void initialize() +{ + static const ECCVerifyHandle ecc_verify_handle; + ECC_Start(); + SelectParams(CBaseChainParams::REGTEST); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::string random_message = fuzzed_data_provider.ConsumeRandomLengthString(1024); + { + const std::vector<uint8_t> random_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); + CKey private_key; + private_key.Set(random_bytes.begin(), random_bytes.end(), fuzzed_data_provider.ConsumeBool()); + std::string signature; + const bool message_signed = MessageSign(private_key, random_message, signature); + if (private_key.IsValid()) { + assert(message_signed); + const MessageVerificationResult verification_result = MessageVerify(EncodeDestination(PKHash(private_key.GetPubKey().GetID())), signature, random_message); + assert(verification_result == MessageVerificationResult::OK); + } + } + { + (void)MessageHash(random_message); + (void)MessageVerify(fuzzed_data_provider.ConsumeRandomLengthString(1024), fuzzed_data_provider.ConsumeRandomLengthString(1024), random_message); + (void)SigningResultString(fuzzed_data_provider.PickValueInArray({SigningResult::OK, SigningResult::PRIVATE_KEY_NOT_AVAILABLE, SigningResult::SIGNING_FAILED})); + } +} diff --git a/src/test/fuzz/net_permissions.cpp b/src/test/fuzz/net_permissions.cpp index bfc5d21427..c071283467 100644 --- a/src/test/fuzz/net_permissions.cpp +++ b/src/test/fuzz/net_permissions.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <net_permissions.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/test/fuzz/parse_hd_keypath.cpp b/src/test/fuzz/parse_hd_keypath.cpp index 9a23f4b2d4..f668ca8c48 100644 --- a/src/test/fuzz/parse_hd_keypath.cpp +++ b/src/test/fuzz/parse_hd_keypath.cpp @@ -2,12 +2,22 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> #include <util/bip32.h> +#include <cstdint> +#include <vector> + void test_one_input(const std::vector<uint8_t>& buffer) { const std::string keypath_str(buffer.begin(), buffer.end()); std::vector<uint32_t> keypath; (void)ParseHDKeypath(keypath_str, keypath); + + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::vector<uint32_t> random_keypath = ConsumeRandomLengthIntegralVector<uint32_t>(fuzzed_data_provider); + (void)FormatHDKeypath(random_keypath); + (void)WriteHDKeypath(random_keypath); } diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp new file mode 100644 index 0000000000..1cbf9b347f --- /dev/null +++ b/src/test/fuzz/policy_estimator.cpp @@ -0,0 +1,69 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <policy/fees.h> +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <txmempool.h> + +#include <cstdint> +#include <optional> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + CBlockPolicyEstimator block_policy_estimator; + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 3)) { + case 0: { + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!mtx) { + break; + } + const CTransaction tx{*mtx}; + block_policy_estimator.processTransaction(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx), fuzzed_data_provider.ConsumeBool()); + if (fuzzed_data_provider.ConsumeBool()) { + (void)block_policy_estimator.removeTx(tx.GetHash(), /* inBlock */ fuzzed_data_provider.ConsumeBool()); + } + break; + } + case 1: { + std::vector<CTxMemPoolEntry> mempool_entries; + while (fuzzed_data_provider.ConsumeBool()) { + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!mtx) { + break; + } + const CTransaction tx{*mtx}; + mempool_entries.push_back(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx)); + } + std::vector<const CTxMemPoolEntry*> ptrs; + ptrs.reserve(mempool_entries.size()); + for (const CTxMemPoolEntry& mempool_entry : mempool_entries) { + ptrs.push_back(&mempool_entry); + } + block_policy_estimator.processBlock(fuzzed_data_provider.ConsumeIntegral<unsigned int>(), ptrs); + break; + } + case 2: { + (void)block_policy_estimator.removeTx(ConsumeUInt256(fuzzed_data_provider), /* inBlock */ fuzzed_data_provider.ConsumeBool()); + break; + } + case 3: { + block_policy_estimator.FlushUnconfirmed(); + break; + } + } + (void)block_policy_estimator.estimateFee(fuzzed_data_provider.ConsumeIntegral<int>()); + EstimationResult result; + (void)block_policy_estimator.estimateRawFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeFloatingPoint<double>(), fuzzed_data_provider.PickValueInArray({FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE}), fuzzed_data_provider.ConsumeBool() ? &result : nullptr); + FeeCalculation fee_calculation; + (void)block_policy_estimator.estimateSmartFee(fuzzed_data_provider.ConsumeIntegral<int>(), fuzzed_data_provider.ConsumeBool() ? &fee_calculation : nullptr, fuzzed_data_provider.ConsumeBool()); + (void)block_policy_estimator.HighestTargetTracked(fuzzed_data_provider.PickValueInArray({FeeEstimateHorizon::SHORT_HALFLIFE, FeeEstimateHorizon::MED_HALFLIFE, FeeEstimateHorizon::LONG_HALFLIFE})); + } +} diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index 0343d33401..b7fc72373d 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -4,7 +4,6 @@ #include <chain.h> #include <chainparams.h> -#include <optional.h> #include <pow.h> #include <primitives/block.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -12,6 +11,7 @@ #include <test/fuzz/util.h> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -28,7 +28,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) const uint32_t fixed_time = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const uint32_t fixed_bits = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); while (fuzzed_data_provider.remaining_bytes() > 0) { - const Optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); + const std::optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); if (!block_header) { continue; } @@ -72,7 +72,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) } } { - const Optional<uint256> hash = ConsumeDeserializable<uint256>(fuzzed_data_provider); + const std::optional<uint256> hash = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (hash) { (void)CheckProofOfWork(*hash, fuzzed_data_provider.ConsumeIntegral<unsigned int>(), consensus_params); } diff --git a/src/test/fuzz/prevector.cpp b/src/test/fuzz/prevector.cpp index 64920f4af5..626e187cbd 100644 --- a/src/test/fuzz/prevector.cpp +++ b/src/test/fuzz/prevector.cpp @@ -14,8 +14,9 @@ namespace { -template<unsigned int N, typename T> -class prevector_tester { +template <unsigned int N, typename T> +class prevector_tester +{ typedef std::vector<T> realtype; realtype real_vector; realtype real_vector_alt; @@ -27,35 +28,36 @@ class prevector_tester { typedef typename pretype::size_type Size; public: - void test() const { + void test() const + { const pretype& const_pre_vector = pre_vector; assert(real_vector.size() == pre_vector.size()); assert(real_vector.empty() == pre_vector.empty()); for (Size s = 0; s < real_vector.size(); s++) { - assert(real_vector[s] == pre_vector[s]); - assert(&(pre_vector[s]) == &(pre_vector.begin()[s])); - assert(&(pre_vector[s]) == &*(pre_vector.begin() + s)); - assert(&(pre_vector[s]) == &*((pre_vector.end() + s) - real_vector.size())); + assert(real_vector[s] == pre_vector[s]); + assert(&(pre_vector[s]) == &(pre_vector.begin()[s])); + assert(&(pre_vector[s]) == &*(pre_vector.begin() + s)); + assert(&(pre_vector[s]) == &*((pre_vector.end() + s) - real_vector.size())); } // assert(realtype(pre_vector) == real_vector); assert(pretype(real_vector.begin(), real_vector.end()) == pre_vector); assert(pretype(pre_vector.begin(), pre_vector.end()) == pre_vector); size_t pos = 0; for (const T& v : pre_vector) { - assert(v == real_vector[pos]); - ++pos; + assert(v == real_vector[pos]); + ++pos; } for (const T& v : reverse_iterate(pre_vector)) { - --pos; - assert(v == real_vector[pos]); + --pos; + assert(v == real_vector[pos]); } for (const T& v : const_pre_vector) { - assert(v == real_vector[pos]); - ++pos; + assert(v == real_vector[pos]); + ++pos; } for (const T& v : reverse_iterate(const_pre_vector)) { - --pos; - assert(v == real_vector[pos]); + --pos; + assert(v == real_vector[pos]); } CDataStream ss1(SER_DISK, 0); CDataStream ss2(SER_DISK, 0); @@ -67,101 +69,120 @@ public: } } - void resize(Size s) { + void resize(Size s) + { real_vector.resize(s); assert(real_vector.size() == s); pre_vector.resize(s); assert(pre_vector.size() == s); } - void reserve(Size s) { + void reserve(Size s) + { real_vector.reserve(s); assert(real_vector.capacity() >= s); pre_vector.reserve(s); assert(pre_vector.capacity() >= s); } - void insert(Size position, const T& value) { + void insert(Size position, const T& value) + { real_vector.insert(real_vector.begin() + position, value); pre_vector.insert(pre_vector.begin() + position, value); } - void insert(Size position, Size count, const T& value) { + void insert(Size position, Size count, const T& value) + { real_vector.insert(real_vector.begin() + position, count, value); pre_vector.insert(pre_vector.begin() + position, count, value); } - template<typename I> - void insert_range(Size position, I first, I last) { + template <typename I> + void insert_range(Size position, I first, I last) + { real_vector.insert(real_vector.begin() + position, first, last); pre_vector.insert(pre_vector.begin() + position, first, last); } - void erase(Size position) { + void erase(Size position) + { real_vector.erase(real_vector.begin() + position); pre_vector.erase(pre_vector.begin() + position); } - void erase(Size first, Size last) { + void erase(Size first, Size last) + { real_vector.erase(real_vector.begin() + first, real_vector.begin() + last); pre_vector.erase(pre_vector.begin() + first, pre_vector.begin() + last); } - void update(Size pos, const T& value) { + void update(Size pos, const T& value) + { real_vector[pos] = value; pre_vector[pos] = value; } - void push_back(const T& value) { + void push_back(const T& value) + { real_vector.push_back(value); pre_vector.push_back(value); } - void pop_back() { + void pop_back() + { real_vector.pop_back(); pre_vector.pop_back(); } - void clear() { + void clear() + { real_vector.clear(); pre_vector.clear(); } - void assign(Size n, const T& value) { + void assign(Size n, const T& value) + { real_vector.assign(n, value); pre_vector.assign(n, value); } - Size size() const { + Size size() const + { return real_vector.size(); } - Size capacity() const { + Size capacity() const + { return pre_vector.capacity(); } - void shrink_to_fit() { + void shrink_to_fit() + { pre_vector.shrink_to_fit(); } - void swap() { + void swap() + { real_vector.swap(real_vector_alt); pre_vector.swap(pre_vector_alt); } - void move() { + void move() + { real_vector = std::move(real_vector_alt); real_vector_alt.clear(); pre_vector = std::move(pre_vector_alt); pre_vector_alt.clear(); } - void copy() { + void copy() + { real_vector = real_vector_alt; pre_vector = pre_vector_alt; } - void resize_uninitialized(realtype values) { + void resize_uninitialized(realtype values) + { size_t r = values.size(); size_t s = real_vector.size() / 2; if (real_vector.capacity() < s + r) { @@ -181,7 +202,7 @@ public: } }; -} +} // namespace void test_one_input(const std::vector<uint8_t>& buffer) { diff --git a/src/test/fuzz/primitives_transaction.cpp b/src/test/fuzz/primitives_transaction.cpp index 2e5ba6bdb0..4a0f920f58 100644 --- a/src/test/fuzz/primitives_transaction.cpp +++ b/src/test/fuzz/primitives_transaction.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <optional.h> #include <primitives/transaction.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -16,7 +16,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CScript script = ConsumeScript(fuzzed_data_provider); - const Optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (out_point) { const CTxIn tx_in{*out_point, script, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; (void)tx_in; @@ -24,8 +24,8 @@ void test_one_input(const std::vector<uint8_t>& buffer) const CTxOut tx_out_1{ConsumeMoney(fuzzed_data_provider), script}; const CTxOut tx_out_2{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; assert((tx_out_1 == tx_out_2) != (tx_out_1 != tx_out_2)); - const Optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); - const Optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (mutable_tx_1 && mutable_tx_2) { const CTransaction tx_1{*mutable_tx_1}; const CTransaction tx_2{*mutable_tx_2}; diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index c03365199a..665a6224b4 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -29,7 +29,7 @@ #include <string> #include <vector> -bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); +bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, ChainstateManager& chainman, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); namespace { @@ -74,7 +74,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) p2p_node.SetSendVersion(PROTOCOL_VERSION); g_setup->m_node.peer_logic->InitializeNode(&p2p_node); try { - (void)ProcessMessage(&p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), Params(), *g_setup->m_node.mempool, g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), std::atomic<bool>{false}); + (void)ProcessMessage(&p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), Params(), *g_setup->m_node.chainman, *g_setup->m_node.mempool, g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), std::atomic<bool>{false}); } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); diff --git a/src/test/fuzz/protocol.cpp b/src/test/fuzz/protocol.cpp index 954471de6c..78df0f89e7 100644 --- a/src/test/fuzz/protocol.cpp +++ b/src/test/fuzz/protocol.cpp @@ -2,20 +2,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <optional.h> #include <protocol.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <cstdint> +#include <optional> #include <stdexcept> #include <vector> void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const Optional<CInv> inv = ConsumeDeserializable<CInv>(fuzzed_data_provider); + const std::optional<CInv> inv = ConsumeDeserializable<CInv>(fuzzed_data_provider); if (!inv) { return; } @@ -24,7 +24,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) } catch (const std::out_of_range&) { } (void)inv->ToString(); - const Optional<CInv> another_inv = ConsumeDeserializable<CInv>(fuzzed_data_provider); + const std::optional<CInv> another_inv = ConsumeDeserializable<CInv>(fuzzed_data_provider); if (!another_inv) { return; } diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp new file mode 100644 index 0000000000..1fd88a5f7b --- /dev/null +++ b/src/test/fuzz/rbf.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <policy/rbf.h> +#include <primitives/transaction.h> +#include <sync.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <txmempool.h> + +#include <cstdint> +#include <optional> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!mtx) { + return; + } + CTxMemPool pool; + while (fuzzed_data_provider.ConsumeBool()) { + const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!another_mtx) { + break; + } + const CTransaction another_tx{*another_mtx}; + if (fuzzed_data_provider.ConsumeBool() && !mtx->vin.empty()) { + mtx->vin[0].prevout = COutPoint{another_tx.GetHash(), 0}; + } + LOCK2(cs_main, pool.cs); + pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, another_tx)); + } + const CTransaction tx{*mtx}; + if (fuzzed_data_provider.ConsumeBool()) { + LOCK2(cs_main, pool.cs); + pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx)); + } + { + LOCK(pool.cs); + (void)IsRBFOptIn(tx, pool); + } +} diff --git a/src/test/fuzz/rolling_bloom_filter.cpp b/src/test/fuzz/rolling_bloom_filter.cpp index 3b37321977..623b8cff3a 100644 --- a/src/test/fuzz/rolling_bloom_filter.cpp +++ b/src/test/fuzz/rolling_bloom_filter.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bloom.h> -#include <optional.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -11,6 +10,7 @@ #include <cassert> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -32,7 +32,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } case 1: { - const Optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); + const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (!u256) { break; } diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index de82122dd6..e0c4ad7eb7 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -21,6 +21,11 @@ #include <univalue.h> #include <util/memory.h> +#include <cstdint> +#include <optional> +#include <string> +#include <vector> + void initialize() { // Fuzzers using pubkey must hold an ECCVerifyHandle. @@ -32,7 +37,7 @@ void initialize() void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const Optional<CScript> script_opt = ConsumeDeserializable<CScript>(fuzzed_data_provider); + const std::optional<CScript> script_opt = ConsumeDeserializable<CScript>(fuzzed_data_provider); if (!script_opt) return; const CScript script{*script_opt}; @@ -101,7 +106,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) } } - const Optional<CScript> other_script = ConsumeDeserializable<CScript>(fuzzed_data_provider); + const std::optional<CScript> other_script = ConsumeDeserializable<CScript>(fuzzed_data_provider); if (other_script) { { CScript script_mut{script}; diff --git a/src/test/fuzz/scriptnum_ops.cpp b/src/test/fuzz/scriptnum_ops.cpp index 42b1432f13..f4e079fb89 100644 --- a/src/test/fuzz/scriptnum_ops.cpp +++ b/src/test/fuzz/scriptnum_ops.cpp @@ -129,10 +129,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) break; } (void)script_num.getint(); - // Avoid negation failure: - // script/script.h:332:35: runtime error: negation of -9223372036854775808 cannot be represented in type 'int64_t' (aka 'long'); cast to an unsigned type to negate this value to itself - if (script_num != CScriptNum{std::numeric_limits<int64_t>::min()}) { - (void)script_num.getvch(); - } + (void)script_num.getvch(); } } diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 3de0cf8db7..50984b1aef 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -93,7 +93,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) { CDataStream data_stream{SER_NETWORK, INIT_PROTO_VERSION}; std::string s; - LimitedString<10> limited_string = LIMITED_STRING(s, 10); + auto limited_string = LIMITED_STRING(s, 10); data_stream << random_string_1; try { data_stream >> limited_string; @@ -108,11 +108,21 @@ void test_one_input(const std::vector<uint8_t>& buffer) } { CDataStream data_stream{SER_NETWORK, INIT_PROTO_VERSION}; - const LimitedString<10> limited_string = LIMITED_STRING(random_string_1, 10); + const auto limited_string = LIMITED_STRING(random_string_1, 10); data_stream << limited_string; std::string deserialized_string; data_stream >> deserialized_string; assert(data_stream.empty()); assert(deserialized_string == random_string_1); } + { + int64_t amount_out; + (void)ParseFixedPoint(random_string_1, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1024), &amount_out); + } + { + (void)Untranslated(random_string_1); + const bilingual_str bs1{random_string_1, random_string_2}; + const bilingual_str bs2{random_string_2, random_string_1}; + (void)(bs1 + bs2); + } } diff --git a/src/test/fuzz/strprintf.cpp b/src/test/fuzz/strprintf.cpp index d5be1070bd..29064bc45c 100644 --- a/src/test/fuzz/strprintf.cpp +++ b/src/test/fuzz/strprintf.cpp @@ -6,6 +6,7 @@ #include <test/fuzz/fuzz.h> #include <tinyformat.h> #include <util/strencodings.h> +#include <util/translation.h> #include <algorithm> #include <cstdint> @@ -16,6 +17,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::string format_string = fuzzed_data_provider.ConsumeRandomLengthString(64); + const bilingual_str bilingual_string{format_string, format_string}; const int digits_in_format_specifier = std::count_if(format_string.begin(), format_string.end(), IsDigit); @@ -47,50 +49,62 @@ void test_one_input(const std::vector<uint8_t>& buffer) try { (void)strprintf(format_string, (signed char*)nullptr); + (void)tinyformat::format(bilingual_string, (signed char*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (unsigned char*)nullptr); + (void)tinyformat::format(bilingual_string, (unsigned char*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (void*)nullptr); + (void)tinyformat::format(bilingual_string, (void*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (bool*)nullptr); + (void)tinyformat::format(bilingual_string, (bool*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (float*)nullptr); + (void)tinyformat::format(bilingual_string, (float*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (double*)nullptr); + (void)tinyformat::format(bilingual_string, (double*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (int16_t*)nullptr); + (void)tinyformat::format(bilingual_string, (int16_t*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (uint16_t*)nullptr); + (void)tinyformat::format(bilingual_string, (uint16_t*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (int32_t*)nullptr); + (void)tinyformat::format(bilingual_string, (int32_t*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (uint32_t*)nullptr); + (void)tinyformat::format(bilingual_string, (uint32_t*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (int64_t*)nullptr); + (void)tinyformat::format(bilingual_string, (int64_t*)nullptr); } catch (const tinyformat::format_error&) { } try { (void)strprintf(format_string, (uint64_t*)nullptr); + (void)tinyformat::format(bilingual_string, (uint64_t*)nullptr); } catch (const tinyformat::format_error&) { } @@ -98,21 +112,27 @@ void test_one_input(const std::vector<uint8_t>& buffer) switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 5)) { case 0: (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); break; case 1: (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); break; case 2: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); break; case 3: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); break; case 4: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<char>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<char>()); break; case 5: (void)strprintf(format_string, fuzzed_data_provider.ConsumeBool()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeBool()); break; } } catch (const tinyformat::format_error&) { @@ -138,27 +158,35 @@ void test_one_input(const std::vector<uint8_t>& buffer) switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 7)) { case 0: (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); break; case 1: (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); break; case 2: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); break; case 3: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); break; case 4: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); break; case 5: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); break; case 6: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); break; case 7: (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + (void)tinyformat::format(bilingual_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); break; } } catch (const tinyformat::format_error&) { diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp new file mode 100644 index 0000000000..01b523cee4 --- /dev/null +++ b/src/test/fuzz/system.cpp @@ -0,0 +1,123 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/system.h> + +#include <cstdint> +#include <string> +#include <vector> + +namespace { +std::string GetArgumentName(const std::string& name) +{ + size_t idx = name.find('='); + if (idx == std::string::npos) { + idx = name.size(); + } + return name.substr(0, idx); +} +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + ArgsManager args_manager{}; + + if (fuzzed_data_provider.ConsumeBool()) { + SetupHelpOptions(args_manager); + } + + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 7)) { + case 0: { + args_manager.SelectConfigNetwork(fuzzed_data_provider.ConsumeRandomLengthString(16)); + break; + } + case 1: { + args_manager.SoftSetArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeRandomLengthString(16)); + break; + } + case 2: { + args_manager.ForceSetArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeRandomLengthString(16)); + break; + } + case 3: { + args_manager.SoftSetBoolArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeBool()); + break; + } + case 4: { + const OptionsCategory options_category = fuzzed_data_provider.PickValueInArray<OptionsCategory>({OptionsCategory::OPTIONS, OptionsCategory::CONNECTION, OptionsCategory::WALLET, OptionsCategory::WALLET_DEBUG_TEST, OptionsCategory::ZMQ, OptionsCategory::DEBUG_TEST, OptionsCategory::CHAINPARAMS, OptionsCategory::NODE_RELAY, OptionsCategory::BLOCK_CREATION, OptionsCategory::RPC, OptionsCategory::GUI, OptionsCategory::COMMANDS, OptionsCategory::REGISTER_COMMANDS, OptionsCategory::HIDDEN}); + // Avoid hitting: + // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + const std::string argument_name = GetArgumentName(fuzzed_data_provider.ConsumeRandomLengthString(16)); + if (args_manager.GetArgFlags(argument_name) != nullopt) { + break; + } + args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), options_category); + break; + } + case 5: { + // Avoid hitting: + // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + const std::vector<std::string> names = ConsumeRandomLengthStringVector(fuzzed_data_provider); + std::vector<std::string> hidden_arguments; + for (const std::string& name : names) { + const std::string hidden_argument = GetArgumentName(name); + if (args_manager.GetArgFlags(hidden_argument) != nullopt) { + continue; + } + if (std::find(hidden_arguments.begin(), hidden_arguments.end(), hidden_argument) != hidden_arguments.end()) { + continue; + } + hidden_arguments.push_back(hidden_argument); + } + args_manager.AddHiddenArgs(hidden_arguments); + break; + } + case 6: { + args_manager.ClearArgs(); + break; + } + case 7: { + const std::vector<std::string> random_arguments = ConsumeRandomLengthStringVector(fuzzed_data_provider); + std::vector<const char*> argv; + argv.reserve(random_arguments.size()); + for (const std::string& random_argument : random_arguments) { + argv.push_back(random_argument.c_str()); + } + try { + std::string error; + (void)args_manager.ParseParameters(argv.size(), argv.data(), error); + } catch (const std::logic_error&) { + } + break; + } + } + } + + const std::string s1 = fuzzed_data_provider.ConsumeRandomLengthString(16); + const std::string s2 = fuzzed_data_provider.ConsumeRandomLengthString(16); + const int64_t i64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const bool b = fuzzed_data_provider.ConsumeBool(); + + (void)args_manager.GetArg(s1, i64); + (void)args_manager.GetArg(s1, s2); + (void)args_manager.GetArgFlags(s1); + (void)args_manager.GetArgs(s1); + (void)args_manager.GetBoolArg(s1, b); + try { + (void)args_manager.GetChainName(); + } catch (const std::runtime_error&) { + } + (void)args_manager.GetHelpMessage(); + (void)args_manager.GetUnrecognizedSections(); + (void)args_manager.GetUnsuitableSectionOnlyArgs(); + (void)args_manager.IsArgNegated(s1); + (void)args_manager.IsArgSet(s1); + + (void)HelpRequested(args_manager); +} diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 9c7b0d47a2..9d0fb02128 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -8,16 +8,20 @@ #include <amount.h> #include <arith_uint256.h> #include <attributes.h> -#include <optional.h> +#include <coins.h> +#include <consensus/consensus.h> +#include <primitives/transaction.h> #include <script/script.h> #include <serialize.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <txmempool.h> #include <uint256.h> #include <version.h> #include <cstdint> +#include <optional> #include <string> #include <vector> @@ -49,7 +53,7 @@ NODISCARD inline std::vector<T> ConsumeRandomLengthIntegralVector(FuzzedDataProv } template <typename T> -NODISCARD inline Optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept +NODISCARD inline std::optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept { const std::vector<uint8_t> buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length); CDataStream ds{buffer, SER_NETWORK, INIT_PROTO_VERSION}; @@ -57,7 +61,7 @@ NODISCARD inline Optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_da try { ds >> obj; } catch (const std::ios_base::failure&) { - return nullopt; + return std::nullopt; } return obj; } @@ -97,6 +101,21 @@ NODISCARD inline arith_uint256 ConsumeArithUInt256(FuzzedDataProvider& fuzzed_da return UintToArith256(ConsumeUInt256(fuzzed_data_provider)); } +NODISCARD inline CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept +{ + // Avoid: + // policy/feerate.cpp:28:34: runtime error: signed integer overflow: 34873208148477500 * 1000 cannot be represented in type 'long' + // + // Reproduce using CFeeRate(348732081484775, 10).GetFeePerK() + const CAmount fee = std::min<CAmount>(ConsumeMoney(fuzzed_data_provider), std::numeric_limits<CAmount>::max() / static_cast<CAmount>(100000)); + assert(MoneyRange(fee)); + const int64_t time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const unsigned int entry_height = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + const bool spends_coinbase = fuzzed_data_provider.ConsumeBool(); + const unsigned int sig_op_cost = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, MAX_BLOCK_SIGOPS_COST); + return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; +} + template <typename T> NODISCARD bool MultiplicationOverflow(const T i, const T j) noexcept { @@ -131,4 +150,15 @@ NODISCARD bool AdditionOverflow(const T i, const T j) noexcept return std::numeric_limits<T>::max() - i < j; } +NODISCARD inline bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept +{ + for (const CTxIn& tx_in : tx.vin) { + const Coin& coin = inputs.AccessCoin(tx_in.prevout); + if (coin.IsSpent()) { + return true; + } + } + return false; +} + #endif // BITCOIN_TEST_FUZZ_UTIL_H diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 9f3ca87206..57eee94330 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -253,7 +253,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) pblock->nNonce = blockinfo[i].nonce; } std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - BOOST_CHECK(ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); + BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); pblock->hashPrevBlock = pblock->GetHash(); } diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index ec6a290334..2e1972cc3f 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -393,7 +393,7 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_CHECK(NetWhitelistPermissions::TryParse("bloom,forcerelay,noban,relay,mempool@1.2.3.4/32", whitelistPermissions, error)); const auto strings = NetPermissions::ToStrings(PF_ALL); - BOOST_CHECK_EQUAL(strings.size(), 5); + BOOST_CHECK_EQUAL(strings.size(), 5U); BOOST_CHECK(std::find(strings.begin(), strings.end(), "bloomfilter") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "forcerelay") != strings.end()); BOOST_CHECK(std::find(strings.begin(), strings.end(), "relay") != strings.end()); diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index ca3b92f2e1..978a7bee4d 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -28,6 +28,8 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests) for (int i = 10; i > 0; --i) { BOOST_CHECK_EQUAL(GetRand(std::numeric_limits<uint64_t>::max()), uint64_t{10393729187455219830U}); BOOST_CHECK_EQUAL(GetRandInt(std::numeric_limits<int>::max()), int{769702006}); + BOOST_CHECK_EQUAL(GetRandMicros(std::chrono::hours{1}).count(), 2917185654); + BOOST_CHECK_EQUAL(GetRandMillis(std::chrono::hours{1}).count(), 2144374); } BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); @@ -47,6 +49,8 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests) for (int i = 10; i > 0; --i) { BOOST_CHECK(GetRand(std::numeric_limits<uint64_t>::max()) != uint64_t{10393729187455219830U}); BOOST_CHECK(GetRandInt(std::numeric_limits<int>::max()) != int{769702006}); + BOOST_CHECK(GetRandMicros(std::chrono::hours{1}) != std::chrono::microseconds{2917185654}); + BOOST_CHECK(GetRandMillis(std::chrono::hours{1}) != std::chrono::milliseconds{2144374}); } { FastRandomContext ctx3, ctx4; @@ -87,7 +91,7 @@ BOOST_AUTO_TEST_CASE(stdrandom_test) BOOST_CHECK(x >= 3); BOOST_CHECK(x <= 9); - std::vector<int> test{1,2,3,4,5,6,7,8,9,10}; + std::vector<int> test{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; std::shuffle(test.begin(), test.end(), ctx); for (int j = 1; j <= 10; ++j) { BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); @@ -97,7 +101,6 @@ BOOST_AUTO_TEST_CASE(stdrandom_test) BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); } } - } /** Test that Shuffle reaches every permutation with equal probability. */ @@ -127,7 +130,7 @@ BOOST_AUTO_TEST_CASE(shuffle_stat_test) } BOOST_CHECK(chi_score > 58.1411); // 99.9999% confidence interval BOOST_CHECK(chi_score < 210.275); - BOOST_CHECK_EQUAL(sum, 12000); + BOOST_CHECK_EQUAL(sum, 12000U); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/ref_tests.cpp b/src/test/ref_tests.cpp new file mode 100644 index 0000000000..0ec0799fbc --- /dev/null +++ b/src/test/ref_tests.cpp @@ -0,0 +1,33 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/ref.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(ref_tests) + +BOOST_AUTO_TEST_CASE(ref_test) +{ + util::Ref ref; + BOOST_CHECK(!ref.Has<int>()); + BOOST_CHECK_THROW(ref.Get<int>(), NonFatalCheckError); + int value = 5; + ref.Set(value); + BOOST_CHECK(ref.Has<int>()); + BOOST_CHECK_EQUAL(ref.Get<int>(), 5); + ++ref.Get<int>(); + BOOST_CHECK_EQUAL(ref.Get<int>(), 6); + BOOST_CHECK_EQUAL(value, 6); + ++value; + BOOST_CHECK_EQUAL(value, 7); + BOOST_CHECK_EQUAL(ref.Get<int>(), 7); + BOOST_CHECK(!ref.Has<bool>()); + BOOST_CHECK_THROW(ref.Get<bool>(), NonFatalCheckError); + ref.Clear(); + BOOST_CHECK(!ref.Has<int>()); + BOOST_CHECK_THROW(ref.Get<int>(), NonFatalCheckError); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index d9c66f1c19..b54cbb3f00 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -10,6 +10,7 @@ #include <interfaces/chain.h> #include <node/context.h> #include <test/util/setup_common.h> +#include <util/ref.h> #include <util/time.h> #include <boost/algorithm/string.hpp> @@ -19,13 +20,20 @@ #include <rpc/blockchain.h> -UniValue CallRPC(std::string args) +class RPCTestingSetup : public TestingSetup +{ +public: + UniValue CallRPC(std::string args); +}; + +UniValue RPCTestingSetup::CallRPC(std::string args) { std::vector<std::string> vArgs; boost::split(vArgs, args, boost::is_any_of(" \t")); std::string strMethod = vArgs[0]; vArgs.erase(vArgs.begin()); - JSONRPCRequest request; + util::Ref context{m_node}; + JSONRPCRequest request(context); request.strMethod = strMethod; request.params = RPCConvertValues(strMethod, vArgs); request.fHelp = false; @@ -40,7 +48,7 @@ UniValue CallRPC(std::string args) } -BOOST_FIXTURE_TEST_SUITE(rpc_tests, TestingSetup) +BOOST_FIXTURE_TEST_SUITE(rpc_tests, RPCTestingSetup) BOOST_AUTO_TEST_CASE(rpc_rawparams) { diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 4d50845256..9a490aaf6b 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -14,7 +14,7 @@ BOOST_AUTO_TEST_CASE(basic_sanity) { BOOST_CHECK_MESSAGE(glibc_sanity_test() == true, "libc sanity test"); BOOST_CHECK_MESSAGE(glibcxx_sanity_test() == true, "stdlib sanity test"); - BOOST_CHECK_MESSAGE(ECC_InitSanityCheck() == true, "openssl ECC test"); + BOOST_CHECK_MESSAGE(ECC_InitSanityCheck() == true, "secp256k1 sanity test"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index de990d9254..b185d3b4ac 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -13,6 +13,12 @@ BOOST_FIXTURE_TEST_SUITE(script_standard_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(dest_default_is_no_dest) +{ + CTxDestination dest; + BOOST_CHECK(!IsValidDestination(dest)); +} + BOOST_AUTO_TEST_CASE(script_standard_Solver_success) { CKey keys[3]; diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 56454f61f3..cb3ae290d1 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -102,7 +102,7 @@ static ScriptErrorDesc script_errors[]={ {SCRIPT_ERR_SIG_FINDANDDELETE, "SIG_FINDANDDELETE"}, }; -static const char *FormatScriptError(ScriptError_t err) +static std::string FormatScriptError(ScriptError_t err) { for (unsigned int i=0; i<ARRAYLEN(script_errors); ++i) if (script_errors[i].err == err) @@ -134,7 +134,7 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript CMutableTransaction tx = BuildSpendingTransaction(scriptSig, scriptWitness, txCredit); CMutableTransaction tx2 = tx; BOOST_CHECK_MESSAGE(VerifyScript(scriptSig, scriptPubKey, &scriptWitness, flags, MutableTransactionSignatureChecker(&tx, 0, txCredit.vout[0].nValue), &err) == expect, message); - BOOST_CHECK_MESSAGE(err == scriptError, std::string(FormatScriptError(err)) + " where " + std::string(FormatScriptError((ScriptError_t)scriptError)) + " expected: " + message); + BOOST_CHECK_MESSAGE(err == scriptError, FormatScriptError(err) + " where " + FormatScriptError((ScriptError_t)scriptError) + " expected: " + message); // Verify that removing flags from a passing test or adding flags to a failing test does not change the result. for (int i = 0; i < 16; ++i) { diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 9a6c721ab8..c2328f931c 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -29,15 +29,13 @@ public: memcpy(charstrval, charstrvalin, sizeof(charstrval)); } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(intval); - READWRITE(boolval); - READWRITE(stringval); - READWRITE(charstrval); - READWRITE(txval); + SERIALIZE_METHODS(CSerializeMethodsTestSingle, obj) + { + READWRITE(obj.intval); + READWRITE(obj.boolval); + READWRITE(obj.stringval); + READWRITE(obj.charstrval); + READWRITE(obj.txval); } bool operator==(const CSerializeMethodsTestSingle& rhs) @@ -54,11 +52,10 @@ class CSerializeMethodsTestMany : public CSerializeMethodsTestSingle { public: using CSerializeMethodsTestSingle::CSerializeMethodsTestSingle; - ADD_SERIALIZE_METHODS; - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(intval, boolval, stringval, charstrval, txval); + SERIALIZE_METHODS(CSerializeMethodsTestMany, obj) + { + READWRITE(obj.intval, obj.boolval, obj.stringval, obj.charstrval, obj.txval); } }; diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 177d8fda73..c509a252e0 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -72,28 +72,28 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6}; VectorReader reader(SER_NETWORK, INIT_PROTO_VERSION, vch, 0); - BOOST_CHECK_EQUAL(reader.size(), 6); + BOOST_CHECK_EQUAL(reader.size(), 6U); BOOST_CHECK(!reader.empty()); // Read a single byte as an unsigned char. unsigned char a; reader >> a; BOOST_CHECK_EQUAL(a, 1); - BOOST_CHECK_EQUAL(reader.size(), 5); + BOOST_CHECK_EQUAL(reader.size(), 5U); BOOST_CHECK(!reader.empty()); // Read a single byte as a signed char. signed char b; reader >> b; BOOST_CHECK_EQUAL(b, -1); - BOOST_CHECK_EQUAL(reader.size(), 4); + BOOST_CHECK_EQUAL(reader.size(), 4U); BOOST_CHECK(!reader.empty()); // Read a 4 bytes as an unsigned int. unsigned int c; reader >> c; - BOOST_CHECK_EQUAL(c, 100992003); // 3,4,5,6 in little-endian base-256 - BOOST_CHECK_EQUAL(reader.size(), 0); + BOOST_CHECK_EQUAL(c, 100992003U); // 3,4,5,6 in little-endian base-256 + BOOST_CHECK_EQUAL(reader.size(), 0U); BOOST_CHECK(reader.empty()); // Reading after end of byte vector throws an error. @@ -104,7 +104,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) VectorReader new_reader(SER_NETWORK, INIT_PROTO_VERSION, vch, 0); new_reader >> d; BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256 - BOOST_CHECK_EQUAL(new_reader.size(), 2); + BOOST_CHECK_EQUAL(new_reader.size(), 2U); BOOST_CHECK(!new_reader.empty()); // Reading after end of byte vector throws an error even if the reader is @@ -136,14 +136,14 @@ BOOST_AUTO_TEST_CASE(bitstream_reader_writer) BOOST_CHECK_EQUAL(serialized_int2, (uint16_t)0x1072); // NOTE: Serialized as LE BitStreamReader<CDataStream> bit_reader(data_copy); - BOOST_CHECK_EQUAL(bit_reader.Read(1), 0); - BOOST_CHECK_EQUAL(bit_reader.Read(2), 2); - BOOST_CHECK_EQUAL(bit_reader.Read(3), 6); - BOOST_CHECK_EQUAL(bit_reader.Read(4), 11); - BOOST_CHECK_EQUAL(bit_reader.Read(5), 1); - BOOST_CHECK_EQUAL(bit_reader.Read(6), 32); - BOOST_CHECK_EQUAL(bit_reader.Read(7), 7); - BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497); + BOOST_CHECK_EQUAL(bit_reader.Read(1), 0U); + BOOST_CHECK_EQUAL(bit_reader.Read(2), 2U); + BOOST_CHECK_EQUAL(bit_reader.Read(3), 6U); + BOOST_CHECK_EQUAL(bit_reader.Read(4), 11U); + BOOST_CHECK_EQUAL(bit_reader.Read(5), 1U); + BOOST_CHECK_EQUAL(bit_reader.Read(6), 32U); + BOOST_CHECK_EQUAL(bit_reader.Read(7), 7U); + BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497U); BOOST_CHECK_THROW(bit_reader.Read(8), std::ios_base::failure); } @@ -236,7 +236,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK_EQUAL(i, 1); // After reading bytes 0 and 1, we're positioned at 2. - BOOST_CHECK_EQUAL(bf.GetPos(), 2); + BOOST_CHECK_EQUAL(bf.GetPos(), 2U); // Rewind to offset 0, ok (within the 10 byte window). BOOST_CHECK(bf.SetPos(0)); @@ -263,18 +263,18 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) // The default argument removes the limit completely. BOOST_CHECK(bf.SetLimit()); // The read position should still be at 3 (no change). - BOOST_CHECK_EQUAL(bf.GetPos(), 3); + BOOST_CHECK_EQUAL(bf.GetPos(), 3U); // Read from current offset, 3, forward until position 10. for (uint8_t j = 3; j < 10; ++j) { bf >> i; BOOST_CHECK_EQUAL(i, j); } - BOOST_CHECK_EQUAL(bf.GetPos(), 10); + BOOST_CHECK_EQUAL(bf.GetPos(), 10U); // We're guaranteed (just barely) to be able to rewind to zero. BOOST_CHECK(bf.SetPos(0)); - BOOST_CHECK_EQUAL(bf.GetPos(), 0); + BOOST_CHECK_EQUAL(bf.GetPos(), 0U); bf >> i; BOOST_CHECK_EQUAL(i, 0); @@ -284,12 +284,12 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK(bf.SetPos(10)); bf >> i; BOOST_CHECK_EQUAL(i, 10); - BOOST_CHECK_EQUAL(bf.GetPos(), 11); + BOOST_CHECK_EQUAL(bf.GetPos(), 11U); // Now it's only guaranteed that we can rewind to offset 1 // (current read position, 11, minus rewind amount, 10). BOOST_CHECK(bf.SetPos(1)); - BOOST_CHECK_EQUAL(bf.GetPos(), 1); + BOOST_CHECK_EQUAL(bf.GetPos(), 1U); bf >> i; BOOST_CHECK_EQUAL(i, 1); @@ -303,7 +303,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK_EQUAL(a[j], 11 + j); } } - BOOST_CHECK_EQUAL(bf.GetPos(), 40); + BOOST_CHECK_EQUAL(bf.GetPos(), 40U); // We've read the entire file, the next read should throw. try { @@ -317,11 +317,11 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK(bf.eof()); // Still at offset 40, we can go back 10, to 30. - BOOST_CHECK_EQUAL(bf.GetPos(), 40); + BOOST_CHECK_EQUAL(bf.GetPos(), 40U); BOOST_CHECK(bf.SetPos(30)); bf >> i; BOOST_CHECK_EQUAL(i, 30); - BOOST_CHECK_EQUAL(bf.GetPos(), 31); + BOOST_CHECK_EQUAL(bf.GetPos(), 31U); // We're too far to rewind to position zero. BOOST_CHECK(!bf.SetPos(0)); diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 9d0ae56c3f..ddbc68f8e2 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -305,7 +305,6 @@ BOOST_AUTO_TEST_CASE(test_Get) t1.vout[0].scriptPubKey << OP_1; BOOST_CHECK(AreInputsStandard(CTransaction(t1), coins)); - BOOST_CHECK_EQUAL(coins.GetValueIn(CTransaction(t1)), (50+21+22)*CENT); } static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const CScript& outscript, CTransactionRef& output, CMutableTransaction& input, bool success = true) diff --git a/src/test/util/logging.cpp b/src/test/util/logging.cpp index fe2e69104b..65a64f2384 100644 --- a/src/test/util/logging.cpp +++ b/src/test/util/logging.cpp @@ -11,13 +11,13 @@ #include <stdexcept> -DebugLogHelper::DebugLogHelper(std::string message) - : m_message{std::move(message)} +DebugLogHelper::DebugLogHelper(std::string message, MatchFn match) + : m_message{std::move(message)}, m_match(std::move(match)) { m_print_connection = LogInstance().PushBackCallback( [this](const std::string& s) { if (m_found) return; - m_found = s.find(m_message) != std::string::npos; + m_found = s.find(m_message) != std::string::npos && m_match(&s); }); noui_test_redirect(); } @@ -26,7 +26,7 @@ void DebugLogHelper::check_found() { noui_reconnect(); LogInstance().DeleteCallback(m_print_connection); - if (!m_found) { + if (!m_found && m_match(nullptr)) { throw std::runtime_error(strprintf("'%s' not found in debug log\n", m_message)); } } diff --git a/src/test/util/logging.h b/src/test/util/logging.h index 45ec44173c..1fcf7ca305 100644 --- a/src/test/util/logging.h +++ b/src/test/util/logging.h @@ -17,10 +17,22 @@ class DebugLogHelper bool m_found{false}; std::list<std::function<void(const std::string&)>>::iterator m_print_connection; + //! Custom match checking function. + //! + //! Invoked with pointers to lines containing matching strings, and with + //! null if check_found() is called without any successful match. + //! + //! Can return true to enable default DebugLogHelper behavior of: + //! (1) ending search after first successful match, and + //! (2) raising an error in check_found if no match was found + //! Can return false to do the opposite in either case. + using MatchFn = std::function<bool(const std::string* line)>; + MatchFn m_match; + void check_found(); public: - DebugLogHelper(std::string message); + DebugLogHelper(std::string message, MatchFn match = [](const std::string*){ return true; }); ~DebugLogHelper() { check_found(); } }; diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index 1df6844062..dac7f1a07b 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -31,7 +31,7 @@ CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) assert(block->nNonce); } - bool processed{ProcessNewBlock(Params(), block, true, nullptr)}; + bool processed{EnsureChainman(node).ProcessNewBlock(Params(), block, true, nullptr)}; assert(processed); return CTxIn{block->vtx[0]->GetHash(), 0}; diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index bf0afc4171..3b7a7c8d12 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -123,7 +123,6 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const const CChainParams& chainparams = Params(); // Ideally we'd move all the RPC tests to the functional testing framework // instead of unit tests, but for now we need these here. - g_rpc_node = &m_node; RegisterAllCoreRPCCommands(tableRPC); m_node.scheduler = MakeUnique<CScheduler>(); @@ -131,11 +130,12 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // We have to run a scheduler thread to prevent ActivateBestChain // from blocking due to queue overrun. threadGroup.create_thread([&]{ m_node.scheduler->serviceQueue(); }); - GetMainSignals().RegisterBackgroundSignalScheduler(*g_rpc_node->scheduler); + GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - g_chainman.InitializeChainstate(); + m_node.chainman = &::g_chainman; + m_node.chainman->InitializeChainstate(); ::ChainstateActive().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!::ChainstateActive().CanFlushToDisk()); @@ -161,7 +161,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const m_node.mempool->setSanityCheck(1.0); m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. - m_node.peer_logic = MakeUnique<PeerLogicValidation>(m_node.connman.get(), m_node.banman.get(), *m_node.scheduler, *m_node.mempool); + m_node.peer_logic = MakeUnique<PeerLogicValidation>(m_node.connman.get(), m_node.banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); { CConnman::Options options; options.m_msgproc = m_node.peer_logic.get(); @@ -176,14 +176,14 @@ TestingSetup::~TestingSetup() threadGroup.join_all(); GetMainSignals().FlushBackgroundCallbacks(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - g_rpc_node = nullptr; m_node.connman.reset(); m_node.banman.reset(); m_node.args = nullptr; m_node.mempool = nullptr; m_node.scheduler.reset(); UnloadBlockIndex(); - g_chainman.Reset(); + m_node.chainman->Reset(); + m_node.chainman = nullptr; pblocktree.reset(); } @@ -228,7 +228,7 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); - ProcessNewBlock(chainparams, shared_pblock, true, nullptr); + EnsureChainman(m_node).ProcessNewBlock(chainparams, shared_pblock, true, nullptr); CBlock result = block; return result; diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 45b7fd4932..cf26ca3adb 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -240,9 +240,9 @@ public: BOOST_CHECK_EQUAL(test.GetSetting("-value").write(), expect.setting.write()); auto settings_list = test.GetSettingsList("-value"); if (expect.setting.isNull() || expect.setting.isFalse()) { - BOOST_CHECK_EQUAL(settings_list.size(), 0); + BOOST_CHECK_EQUAL(settings_list.size(), 0U); } else { - BOOST_CHECK_EQUAL(settings_list.size(), 1); + BOOST_CHECK_EQUAL(settings_list.size(), 1U); BOOST_CHECK_EQUAL(settings_list[0].write(), expect.setting.write()); } @@ -1911,7 +1911,7 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) input = "xxx"; results = Split(MakeSpan(input), 'x'); - BOOST_CHECK_EQUAL(results.size(), 4); + BOOST_CHECK_EQUAL(results.size(), 4U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), ""); BOOST_CHECK_EQUAL(SpanToStr(results[1]), ""); BOOST_CHECK_EQUAL(SpanToStr(results[2]), ""); @@ -1919,19 +1919,19 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) input = "one#two#three"; results = Split(MakeSpan(input), '-'); - BOOST_CHECK_EQUAL(results.size(), 1); + BOOST_CHECK_EQUAL(results.size(), 1U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), "one#two#three"); input = "one#two#three"; results = Split(MakeSpan(input), '#'); - BOOST_CHECK_EQUAL(results.size(), 3); + BOOST_CHECK_EQUAL(results.size(), 3U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), "one"); BOOST_CHECK_EQUAL(SpanToStr(results[1]), "two"); BOOST_CHECK_EQUAL(SpanToStr(results[2]), "three"); input = "*foo*bar*"; results = Split(MakeSpan(input), '*'); - BOOST_CHECK_EQUAL(results.size(), 4); + BOOST_CHECK_EQUAL(results.size(), 4U); BOOST_CHECK_EQUAL(SpanToStr(results[0]), ""); BOOST_CHECK_EQUAL(SpanToStr(results[1]), "foo"); BOOST_CHECK_EQUAL(SpanToStr(results[2]), "bar"); @@ -1990,24 +1990,24 @@ BOOST_AUTO_TEST_CASE(test_tracked_vector) BOOST_CHECK(t3.origin == &t3); auto v1 = Vector(t1); - BOOST_CHECK_EQUAL(v1.size(), 1); + BOOST_CHECK_EQUAL(v1.size(), 1U); BOOST_CHECK(v1[0].origin == &t1); BOOST_CHECK_EQUAL(v1[0].copies, 1); auto v2 = Vector(std::move(t2)); - BOOST_CHECK_EQUAL(v2.size(), 1); + BOOST_CHECK_EQUAL(v2.size(), 1U); BOOST_CHECK(v2[0].origin == &t2); BOOST_CHECK_EQUAL(v2[0].copies, 0); auto v3 = Vector(t1, std::move(t2)); - BOOST_CHECK_EQUAL(v3.size(), 2); + BOOST_CHECK_EQUAL(v3.size(), 2U); BOOST_CHECK(v3[0].origin == &t1); BOOST_CHECK(v3[1].origin == &t2); BOOST_CHECK_EQUAL(v3[0].copies, 1); BOOST_CHECK_EQUAL(v3[1].copies, 0); auto v4 = Vector(std::move(v3[0]), v3[1], std::move(t3)); - BOOST_CHECK_EQUAL(v4.size(), 3); + BOOST_CHECK_EQUAL(v4.size(), 3U); BOOST_CHECK(v4[0].origin == &t1); BOOST_CHECK(v4[1].origin == &t2); BOOST_CHECK(v4[2].origin == &t3); @@ -2016,7 +2016,7 @@ BOOST_AUTO_TEST_CASE(test_tracked_vector) BOOST_CHECK_EQUAL(v4[2].copies, 0); auto v5 = Cat(v1, v4); - BOOST_CHECK_EQUAL(v5.size(), 4); + BOOST_CHECK_EQUAL(v5.size(), 4U); BOOST_CHECK(v5[0].origin == &t1); BOOST_CHECK(v5[1].origin == &t1); BOOST_CHECK(v5[2].origin == &t2); @@ -2027,7 +2027,7 @@ BOOST_AUTO_TEST_CASE(test_tracked_vector) BOOST_CHECK_EQUAL(v5[3].copies, 1); auto v6 = Cat(std::move(v1), v3); - BOOST_CHECK_EQUAL(v6.size(), 3); + BOOST_CHECK_EQUAL(v6.size(), 3U); BOOST_CHECK(v6[0].origin == &t1); BOOST_CHECK(v6[1].origin == &t1); BOOST_CHECK(v6[2].origin == &t2); @@ -2036,7 +2036,7 @@ BOOST_AUTO_TEST_CASE(test_tracked_vector) BOOST_CHECK_EQUAL(v6[2].copies, 1); auto v7 = Cat(v2, std::move(v4)); - BOOST_CHECK_EQUAL(v7.size(), 4); + BOOST_CHECK_EQUAL(v7.size(), 4U); BOOST_CHECK(v7[0].origin == &t2); BOOST_CHECK(v7[1].origin == &t1); BOOST_CHECK(v7[2].origin == &t2); @@ -2047,7 +2047,7 @@ BOOST_AUTO_TEST_CASE(test_tracked_vector) BOOST_CHECK_EQUAL(v7[3].copies, 0); auto v8 = Cat(std::move(v2), std::move(v3)); - BOOST_CHECK_EQUAL(v8.size(), 3); + BOOST_CHECK_EQUAL(v8.size(), 3U); BOOST_CHECK(v8[0].origin == &t2); BOOST_CHECK(v8[1].origin == &t1); BOOST_CHECK(v8[2].origin == &t2); diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index f226caf717..4dcc080b2d 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -62,7 +62,7 @@ BOOST_AUTO_TEST_CASE(util_threadnames_test_rename_threaded) std::set<std::string> names = RenameEnMasse(100); - BOOST_CHECK_EQUAL(names.size(), 100); + BOOST_CHECK_EQUAL(names.size(), 100U); // Names "test_thread.[n]" should exist for n = [0, 99] for (int i = 0; i < 100; ++i) { diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index b8af869b8e..45e0c5484e 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -32,7 +32,7 @@ struct MinerTestingSetup : public RegTestingSetup { BOOST_FIXTURE_TEST_SUITE(validation_block_tests, MinerTestingSetup) -struct TestSubscriber : public CValidationInterface { +struct TestSubscriber final : public CValidationInterface { uint256 m_expected_tip; explicit TestSubscriber(uint256 tip) : m_expected_tip(tip) {} @@ -163,10 +163,10 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) std::transform(blocks.begin(), blocks.end(), std::back_inserter(headers), [](std::shared_ptr<const CBlock> b) { return b->GetBlockHeader(); }); // Process all the headers so we understand the toplogy of the chain - BOOST_CHECK(ProcessNewBlockHeaders(headers, state, Params())); + BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlockHeaders(headers, state, Params())); // Connect the genesis block and drain any outstanding events - BOOST_CHECK(ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); + BOOST_CHECK(EnsureChainman(m_node).ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); SyncWithValidationInterfaceQueue(); // subscribe to events (this subscriber will validate event ordering) @@ -175,26 +175,26 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) LOCK(cs_main); initial_tip = ::ChainActive().Tip(); } - TestSubscriber sub(initial_tip->GetBlockHash()); - RegisterValidationInterface(&sub); + auto sub = std::make_shared<TestSubscriber>(initial_tip->GetBlockHash()); + RegisterSharedValidationInterface(sub); // create a bunch of threads that repeatedly process a block generated above at random // this will create parallelism and randomness inside validation - the ValidationInterface // will subscribe to events generated during block validation and assert on ordering invariance std::vector<std::thread> threads; for (int i = 0; i < 10; i++) { - threads.emplace_back([&blocks]() { + threads.emplace_back([&]() { bool ignored; FastRandomContext insecure; for (int i = 0; i < 1000; i++) { auto block = blocks[insecure.randrange(blocks.size() - 1)]; - ProcessNewBlock(Params(), block, true, &ignored); + EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, &ignored); } // to make sure that eventually we process the full chain - do it here for (auto block : blocks) { if (block->vtx.size() == 1) { - bool processed = ProcessNewBlock(Params(), block, true, &ignored); + bool processed = EnsureChainman(m_node).ProcessNewBlock(Params(), block, true, &ignored); assert(processed); } } @@ -204,14 +204,12 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) for (auto& t : threads) { t.join(); } - while (GetMainSignals().CallbacksPending() > 0) { - UninterruptibleSleep(std::chrono::milliseconds{100}); - } + SyncWithValidationInterfaceQueue(); - UnregisterValidationInterface(&sub); + UnregisterSharedValidationInterface(sub); LOCK(cs_main); - BOOST_CHECK_EQUAL(sub.m_expected_tip, ::ChainActive().Tip()->GetBlockHash()); + BOOST_CHECK_EQUAL(sub->m_expected_tip, ::ChainActive().Tip()->GetBlockHash()); } /** @@ -234,8 +232,8 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) BOOST_AUTO_TEST_CASE(mempool_locks_reorg) { bool ignored; - auto ProcessBlock = [&ignored](std::shared_ptr<const CBlock> block) -> bool { - return ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); + auto ProcessBlock = [&](std::shared_ptr<const CBlock> block) -> bool { + return EnsureChainman(m_node).ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); }; // Process all mined blocks @@ -340,4 +338,38 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) rpc_thread.join(); } } + +BOOST_AUTO_TEST_CASE(witness_commitment_index) +{ + CScript pubKey; + pubKey << 1 << OP_TRUE; + auto ptemplate = BlockAssembler(*m_node.mempool, Params()).CreateNewBlock(pubKey); + CBlock pblock = ptemplate->block; + + CTxOut witness; + witness.scriptPubKey.resize(MINIMUM_WITNESS_COMMITMENT); + witness.scriptPubKey[0] = OP_RETURN; + witness.scriptPubKey[1] = 0x24; + witness.scriptPubKey[2] = 0xaa; + witness.scriptPubKey[3] = 0x21; + witness.scriptPubKey[4] = 0xa9; + witness.scriptPubKey[5] = 0xed; + + // A witness larger than the minimum size is still valid + CTxOut min_plus_one = witness; + min_plus_one.scriptPubKey.resize(MINIMUM_WITNESS_COMMITMENT + 1); + + CTxOut invalid = witness; + invalid.scriptPubKey[0] = OP_VERIFY; + + CMutableTransaction txCoinbase(*pblock.vtx[0]); + txCoinbase.vout.resize(4); + txCoinbase.vout[0] = witness; + txCoinbase.vout[1] = witness; + txCoinbase.vout[2] = min_plus_one; + txCoinbase.vout[3] = invalid; + pblock.vtx[0] = MakeTransactionRef(std::move(txCoinbase)); + + BOOST_CHECK_EQUAL(GetWitnessCommitmentIndex(pblock), 2); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index 388a2dbd13..a863e3a4d5 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -46,7 +46,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // (prevector<28, unsigned char>) when assigned 56 bytes of data per above. // // See also: Coin::DynamicMemoryUsage(). - constexpr int COIN_SIZE = is_64_bit ? 80 : 64; + constexpr unsigned int COIN_SIZE = is_64_bit ? 80 : 64; auto print_view_mem_usage = [](CCoinsViewCache& view) { BOOST_TEST_MESSAGE("CCoinsViewCache memory usage: " << view.DynamicMemoryUsage()); @@ -79,7 +79,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) } print_view_mem_usage(view); - BOOST_CHECK_EQUAL(view.DynamicMemoryUsage(), is_64_bit ? 32 : 16); + BOOST_CHECK_EQUAL(view.DynamicMemoryUsage(), is_64_bit ? 32U : 16U); // We should be able to add COINS_UNTIL_CRITICAL coins to the cache before going CRITICAL. // This is contingent not only on the dynamic memory usage of the Coins diff --git a/src/test/validationinterface_tests.cpp b/src/test/validationinterface_tests.cpp index 208be92852..ceba689e52 100644 --- a/src/test/validationinterface_tests.cpp +++ b/src/test/validationinterface_tests.cpp @@ -12,6 +12,40 @@ BOOST_FIXTURE_TEST_SUITE(validationinterface_tests, TestingSetup) +struct TestSubscriberNoop final : public CValidationInterface { + void BlockChecked(const CBlock&, const BlockValidationState&) override {} +}; + +BOOST_AUTO_TEST_CASE(unregister_validation_interface_race) +{ + std::atomic<bool> generate{true}; + + // Start thread to generate notifications + std::thread gen{[&] { + const CBlock block_dummy; + BlockValidationState state_dummy; + while (generate) { + GetMainSignals().BlockChecked(block_dummy, state_dummy); + } + }}; + + // Start thread to consume notifications + std::thread sub{[&] { + // keep going for about 1 sec, which is 250k iterations + for (int i = 0; i < 250000; i++) { + auto sub = std::make_shared<TestSubscriberNoop>(); + RegisterSharedValidationInterface(sub); + UnregisterSharedValidationInterface(sub); + } + // tell the other thread we are done + generate = false; + }}; + + gen.join(); + sub.join(); + BOOST_CHECK(!generate); +} + class TestInterface : public CValidationInterface { public: diff --git a/src/threadsafety.h b/src/threadsafety.h index bb988dfdfd..942aa3fdcd 100644 --- a/src/threadsafety.h +++ b/src/threadsafety.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_THREADSAFETY_H #define BITCOIN_THREADSAFETY_H +#include <mutex> + #ifdef __clang__ // TL;DR Add GUARDED_BY(mutex) to member variables. The others are // rarely necessary. Ex: int nFoo GUARDED_BY(cs_foo); @@ -54,4 +56,19 @@ #define ASSERT_EXCLUSIVE_LOCK(...) #endif // __GNUC__ +// StdMutex provides an annotated version of std::mutex for us, +// and should only be used when sync.h Mutex/LOCK/etc are not usable. +class LOCKABLE StdMutex : public std::mutex +{ +}; + +// StdLockGuard provides an annotated version of std::lock_guard for us, +// and should only be used when sync.h Mutex/LOCK/etc are not usable. +class SCOPED_LOCKABLE StdLockGuard : public std::lock_guard<StdMutex> +{ +public: + explicit StdLockGuard(StdMutex& cs) EXCLUSIVE_LOCK_FUNCTION(cs) : std::lock_guard<StdMutex>(cs) {} + ~StdLockGuard() UNLOCK_FUNCTION() {} +}; + #endif // BITCOIN_THREADSAFETY_H diff --git a/src/timedata.cpp b/src/timedata.cpp index 942b3cb919..dd56acf44f 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -101,8 +101,8 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) if (!fMatch) { fDone = true; - std::string strMessage = strprintf(_("Please check that your computer's date and time are correct! If your clock is wrong, %s will not work properly.").translated, PACKAGE_NAME); - SetMiscWarning(strMessage); + bilingual_str strMessage = strprintf(_("Please check that your computer's date and time are correct! If your clock is wrong, %s will not work properly."), PACKAGE_NAME); + SetMiscWarning(strMessage.translated); uiInterface.ThreadSafeMessageBox(strMessage, "", CClientUIInterface::MSG_WARNING); } } diff --git a/src/txdb.cpp b/src/txdb.cpp index 071aa1336b..129697f0e7 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -36,19 +36,7 @@ struct CoinEntry { char key; explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN) {} - template<typename Stream> - void Serialize(Stream &s) const { - s << key; - s << outpoint->hash; - s << VARINT(outpoint->n); - } - - template<typename Stream> - void Unserialize(Stream& s) { - s >> key; - s >> outpoint->hash; - s >> VARINT(outpoint->n); - } + SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); } }; } diff --git a/src/txmempool.h b/src/txmempool.h index 4bee78b8d6..4568eb928d 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -704,7 +704,10 @@ public: /** Adds a transaction to the unbroadcast set */ void AddUnbroadcastTx(const uint256& txid) { LOCK(cs); - m_unbroadcast_txids.insert(txid); + /** Sanity Check: the transaction should also be in the mempool */ + if (exists(txid)) { + m_unbroadcast_txids.insert(txid); + } } /** Removes a transaction from the unbroadcast set */ @@ -716,6 +719,12 @@ public: return m_unbroadcast_txids; } + // Returns if a txid is in the unbroadcast set + bool IsUnbroadcastTx(const uint256& txid) const { + LOCK(cs); + return (m_unbroadcast_txids.count(txid) != 0); + } + private: /** UpdateForDescendants is used by UpdateTransactionsFromBlock to update * the descendants for a single transaction that has been added to the diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp index 85bb746d19..15795bd67f 100644 --- a/src/ui_interface.cpp +++ b/src/ui_interface.cpp @@ -4,6 +4,8 @@ #include <ui_interface.h> +#include <util/translation.h> + #include <boost/signals2/last_value.hpp> #include <boost/signals2/signal.hpp> @@ -40,25 +42,24 @@ ADD_SIGNALS_IMPL_WRAPPER(NotifyBlockTip); ADD_SIGNALS_IMPL_WRAPPER(NotifyHeaderTip); ADD_SIGNALS_IMPL_WRAPPER(BannedListChanged); -bool CClientUIInterface::ThreadSafeMessageBox(const std::string& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style); } -bool CClientUIInterface::ThreadSafeQuestion(const std::string& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style); } +bool CClientUIInterface::ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style); } +bool CClientUIInterface::ThreadSafeQuestion(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style); } void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_signals.InitMessage(message); } void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } void CClientUIInterface::NotifyAlertChanged() { return g_ui_signals.NotifyAlertChanged(); } void CClientUIInterface::ShowProgress(const std::string& title, int nProgress, bool resume_possible) { return g_ui_signals.ShowProgress(title, nProgress, resume_possible); } -void CClientUIInterface::NotifyBlockTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyBlockTip(b, i); } -void CClientUIInterface::NotifyHeaderTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyHeaderTip(b, i); } +void CClientUIInterface::NotifyBlockTip(SynchronizationState s, const CBlockIndex* i) { return g_ui_signals.NotifyBlockTip(s, i); } +void CClientUIInterface::NotifyHeaderTip(SynchronizationState s, const CBlockIndex* i) { return g_ui_signals.NotifyHeaderTip(s, i); } void CClientUIInterface::BannedListChanged() { return g_ui_signals.BannedListChanged(); } - -bool InitError(const std::string& str) +bool InitError(const bilingual_str& str) { uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_ERROR); return false; } -void InitWarning(const std::string& str) +void InitWarning(const bilingual_str& str) { uiInterface.ThreadSafeMessageBox(str, "", CClientUIInterface::MSG_WARNING); } diff --git a/src/ui_interface.h b/src/ui_interface.h index b402177b85..d45811178f 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -11,6 +11,9 @@ #include <string> class CBlockIndex; +enum class SynchronizationState; +struct bilingual_str; + namespace boost { namespace signals2 { class connection; @@ -82,10 +85,10 @@ public: boost::signals2::connection signal_name##_connect(std::function<signal_name##Sig> fn); /** Show message box. */ - ADD_SIGNALS_DECL_WRAPPER(ThreadSafeMessageBox, bool, const std::string& message, const std::string& caption, unsigned int style); + ADD_SIGNALS_DECL_WRAPPER(ThreadSafeMessageBox, bool, const bilingual_str& message, const std::string& caption, unsigned int style); /** If possible, ask the user a question. If not, falls back to ThreadSafeMessageBox(noninteractive_message, caption, style) and returns false. */ - ADD_SIGNALS_DECL_WRAPPER(ThreadSafeQuestion, bool, const std::string& message, const std::string& noninteractive_message, const std::string& caption, unsigned int style); + ADD_SIGNALS_DECL_WRAPPER(ThreadSafeQuestion, bool, const bilingual_str& message, const std::string& noninteractive_message, const std::string& caption, unsigned int style); /** Progress message during initialization. */ ADD_SIGNALS_DECL_WRAPPER(InitMessage, void, const std::string& message); @@ -108,20 +111,20 @@ public: ADD_SIGNALS_DECL_WRAPPER(ShowProgress, void, const std::string& title, int nProgress, bool resume_possible); /** New block has been accepted */ - ADD_SIGNALS_DECL_WRAPPER(NotifyBlockTip, void, bool, const CBlockIndex*); + ADD_SIGNALS_DECL_WRAPPER(NotifyBlockTip, void, SynchronizationState, const CBlockIndex*); /** Best header has changed */ - ADD_SIGNALS_DECL_WRAPPER(NotifyHeaderTip, void, bool, const CBlockIndex*); + ADD_SIGNALS_DECL_WRAPPER(NotifyHeaderTip, void, SynchronizationState, const CBlockIndex*); /** Banlist did change. */ ADD_SIGNALS_DECL_WRAPPER(BannedListChanged, void, void); }; /** Show warning message **/ -void InitWarning(const std::string& str); +void InitWarning(const bilingual_str& str); /** Show error message **/ -bool InitError(const std::string& str); +bool InitError(const bilingual_str& str); extern CClientUIInterface uiInterface; diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index 5354cdb962..bd77d74218 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -2,12 +2,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <map> #include <vector> #include <assert.h> #include <crypto/common.h> namespace { +constexpr uint32_t INVALID = 0xFFFFFFFF; + uint32_t DecodeBits(std::vector<bool>::const_iterator& bitpos, const std::vector<bool>::const_iterator& endpos, uint8_t minval, const std::vector<uint8_t> &bit_sizes) { uint32_t val = minval; @@ -25,7 +28,7 @@ uint32_t DecodeBits(std::vector<bool>::const_iterator& bitpos, const std::vector val += (1 << *bit_sizes_it); } else { for (int b = 0; b < *bit_sizes_it; b++) { - if (bitpos == endpos) break; + if (bitpos == endpos) return INVALID; // Reached EOF in mantissa bit = *bitpos; bitpos++; val += bit << (*bit_sizes_it - 1 - b); @@ -33,13 +36,21 @@ uint32_t DecodeBits(std::vector<bool>::const_iterator& bitpos, const std::vector return val; } } - return -1; + return INVALID; // Reached EOF in exponent } +enum class Instruction : uint32_t +{ + RETURN = 0, + JUMP = 1, + MATCH = 2, + DEFAULT = 3, +}; + const std::vector<uint8_t> TYPE_BIT_SIZES{0, 0, 1}; -uint32_t DecodeType(std::vector<bool>::const_iterator& bitpos, const std::vector<bool>::const_iterator& endpos) +Instruction DecodeType(std::vector<bool>::const_iterator& bitpos, const std::vector<bool>::const_iterator& endpos) { - return DecodeBits(bitpos, endpos, 0, TYPE_BIT_SIZES); + return Instruction(DecodeBits(bitpos, endpos, 0, TYPE_BIT_SIZES)); } const std::vector<uint8_t> ASN_BIT_SIZES{15, 16, 17, 18, 19, 20, 21, 22, 23, 24}; @@ -70,34 +81,107 @@ uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip) const std::vector<bool>::const_iterator endpos = asmap.end(); uint8_t bits = ip.size(); uint32_t default_asn = 0; - uint32_t opcode, jump, match, matchlen; + uint32_t jump, match, matchlen; + Instruction opcode; while (pos != endpos) { opcode = DecodeType(pos, endpos); - if (opcode == 0) { - return DecodeASN(pos, endpos); - } else if (opcode == 1) { + if (opcode == Instruction::RETURN) { + default_asn = DecodeASN(pos, endpos); + if (default_asn == INVALID) break; // ASN straddles EOF + return default_asn; + } else if (opcode == Instruction::JUMP) { jump = DecodeJump(pos, endpos); - if (bits == 0) break; + if (jump == INVALID) break; // Jump offset straddles EOF + if (bits == 0) break; // No input bits left + if (pos + jump < pos) break; // overflow + if (pos + jump >= endpos) break; // Jumping past EOF if (ip[ip.size() - bits]) { - if (jump >= endpos - pos) break; pos += jump; } bits--; - } else if (opcode == 2) { + } else if (opcode == Instruction::MATCH) { match = DecodeMatch(pos, endpos); + if (match == INVALID) break; // Match bits straddle EOF matchlen = CountBits(match) - 1; + if (bits < matchlen) break; // Not enough input bits for (uint32_t bit = 0; bit < matchlen; bit++) { - if (bits == 0) break; if ((ip[ip.size() - bits]) != ((match >> (matchlen - 1 - bit)) & 1)) { return default_asn; } bits--; } - } else if (opcode == 3) { + } else if (opcode == Instruction::DEFAULT) { default_asn = DecodeASN(pos, endpos); + if (default_asn == INVALID) break; // ASN straddles EOF } else { - break; + break; // Instruction straddles EOF } } + assert(false); // Reached EOF without RETURN, or aborted (see any of the breaks above) - should have been caught by SanityCheckASMap below return 0; // 0 is not a valid ASN } + +bool SanityCheckASMap(const std::vector<bool>& asmap, int bits) +{ + const std::vector<bool>::const_iterator begin = asmap.begin(), endpos = asmap.end(); + std::vector<bool>::const_iterator pos = begin; + std::vector<std::pair<uint32_t, int>> jumps; // All future positions we may jump to (bit offset in asmap -> bits to consume left) + jumps.reserve(bits); + Instruction prevopcode = Instruction::JUMP; + bool had_incomplete_match = false; + while (pos != endpos) { + uint32_t offset = pos - begin; + if (!jumps.empty() && offset >= jumps.back().first) return false; // There was a jump into the middle of the previous instruction + Instruction opcode = DecodeType(pos, endpos); + if (opcode == Instruction::RETURN) { + if (prevopcode == Instruction::DEFAULT) return false; // There should not be any RETURN immediately after a DEFAULT (could be combined into just RETURN) + uint32_t asn = DecodeASN(pos, endpos); + if (asn == INVALID) return false; // ASN straddles EOF + if (jumps.empty()) { + // Nothing to execute anymore + if (endpos - pos > 7) return false; // Excessive padding + while (pos != endpos) { + if (*pos) return false; // Nonzero padding bit + ++pos; + } + return true; // Sanely reached EOF + } else { + // Continue by pretending we jumped to the next instruction + offset = pos - begin; + if (offset != jumps.back().first) return false; // Unreachable code + bits = jumps.back().second; // Restore the number of bits we would have had left after this jump + jumps.pop_back(); + prevopcode = Instruction::JUMP; + } + } else if (opcode == Instruction::JUMP) { + uint32_t jump = DecodeJump(pos, endpos); + if (jump == INVALID) return false; // Jump offset straddles EOF + if (pos + jump < pos) return false; // overflow + if (pos + jump > endpos) return false; // Jump out of range + if (bits == 0) return false; // Consuming bits past the end of the input + --bits; + uint32_t jump_offset = pos - begin + jump; + if (!jumps.empty() && jump_offset >= jumps.back().first) return false; // Intersecting jumps + jumps.emplace_back(jump_offset, bits); + prevopcode = Instruction::JUMP; + } else if (opcode == Instruction::MATCH) { + uint32_t match = DecodeMatch(pos, endpos); + if (match == INVALID) return false; // Match bits straddle EOF + int matchlen = CountBits(match) - 1; + if (prevopcode != Instruction::MATCH) had_incomplete_match = false; + if (matchlen < 8 && had_incomplete_match) return false; // Within a sequence of matches only at most one should be incomplete + had_incomplete_match = (matchlen < 8); + if (bits < matchlen) return false; // Consuming bits past the end of the input + bits -= matchlen; + prevopcode = Instruction::MATCH; + } else if (opcode == Instruction::DEFAULT) { + if (prevopcode == Instruction::DEFAULT) return false; // There should not be two successive DEFAULTs (they could be combined into one) + uint32_t asn = DecodeASN(pos, endpos); + if (asn == INVALID) return false; // ASN straddles EOF + prevopcode = Instruction::DEFAULT; + } else { + return false; // Instruction straddles EOF + } + } + return false; // Reached EOF without RETURN instruction +} diff --git a/src/util/asmap.h b/src/util/asmap.h index a0e14013c5..b31e639bb5 100644 --- a/src/util/asmap.h +++ b/src/util/asmap.h @@ -5,6 +5,11 @@ #ifndef BITCOIN_UTIL_ASMAP_H #define BITCOIN_UTIL_ASMAP_H +#include <stdint.h> +#include <vector> + uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip); +bool SanityCheckASMap(const std::vector<bool>& asmap, int bits); + #endif // BITCOIN_UTIL_ASMAP_H diff --git a/src/util/check.h b/src/util/check.h index d18887ae95..5c0f32cf51 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -5,6 +5,10 @@ #ifndef BITCOIN_UTIL_CHECK_H #define BITCOIN_UTIL_CHECK_H +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + #include <tinyformat.h> #include <stdexcept> diff --git a/src/util/ref.h b/src/util/ref.h new file mode 100644 index 0000000000..9685ea9fec --- /dev/null +++ b/src/util/ref.h @@ -0,0 +1,38 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_REF_H +#define BITCOIN_UTIL_REF_H + +#include <util/check.h> + +#include <typeindex> + +namespace util { + +/** + * Type-safe dynamic reference. + * + * This implements a small subset of the functionality in C++17's std::any + * class, and can be dropped when the project updates to C++17 + * (https://github.com/bitcoin/bitcoin/issues/16684) + */ +class Ref +{ +public: + Ref() = default; + template<typename T> Ref(T& value) { Set(value); } + template<typename T> T& Get() const { CHECK_NONFATAL(Has<T>()); return *static_cast<T*>(m_value); } + template<typename T> void Set(T& value) { m_value = &value; m_type = std::type_index(typeid(T)); } + template<typename T> bool Has() const { return m_value && m_type == std::type_index(typeid(T)); } + void Clear() { m_value = nullptr; m_type = std::type_index(typeid(void)); } + +private: + void* m_value = nullptr; + std::type_index m_type = std::type_index(typeid(void)); +}; + +} // namespace util + +#endif // BITCOIN_UTIL_REF_H diff --git a/src/util/string.h b/src/util/string.h index b8e2a06235..cdb41630c6 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -30,10 +30,11 @@ NODISCARD inline std::string TrimString(const std::string& str, const std::strin * @param separator The separator * @param unary_op Apply this operator to each item in the list */ -template <typename T, typename UnaryOp> -std::string Join(const std::vector<T>& list, const std::string& separator, UnaryOp unary_op) +template <typename T, typename BaseType, typename UnaryOp> +auto Join(const std::vector<T>& list, const BaseType& separator, UnaryOp unary_op) + -> decltype(unary_op(list.at(0))) { - std::string ret; + decltype(unary_op(list.at(0))) ret; for (size_t i = 0; i < list.size(); ++i) { if (i > 0) ret += separator; ret += unary_op(list.at(i)); @@ -41,9 +42,16 @@ std::string Join(const std::vector<T>& list, const std::string& separator, Unary return ret; } +template <typename T> +T Join(const std::vector<T>& list, const T& separator) +{ + return Join(list, separator, [](const T& i) { return i; }); +} + +// Explicit overload needed for c_str arguments, which would otherwise cause a substitution failure in the template above. inline std::string Join(const std::vector<std::string>& list, const std::string& separator) { - return Join(list, separator, [](const std::string& i) { return i; }); + return Join<std::string>(list, separator); } /** diff --git a/src/util/system.cpp b/src/util/system.cpp index 0790ea0d48..bde0f097be 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <sync.h> #include <util/system.h> #include <chainparamsbase.h> @@ -17,7 +18,7 @@ #endif #ifndef WIN32 -// for posix_fallocate +// for posix_fallocate, in configure.ac we check if it is present after this #ifdef __linux__ #ifdef _POSIX_C_SOURCE @@ -75,18 +76,18 @@ const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; ArgsManager gArgs; +/** Mutex to protect dir_locks. */ +static Mutex cs_dir_locks; /** A map that contains all the currently held directory locks. After * successful locking, these will be held here until the global destructor * cleans them up and thus automatically unlocks them, or ReleaseDirectoryLocks * is called. */ -static std::map<std::string, std::unique_ptr<fsbridge::FileLock>> dir_locks; -/** Mutex to protect dir_locks. */ -static std::mutex cs_dir_locks; +static std::map<std::string, std::unique_ptr<fsbridge::FileLock>> dir_locks GUARDED_BY(cs_dir_locks); bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only) { - std::lock_guard<std::mutex> ulock(cs_dir_locks); + LOCK(cs_dir_locks); fs::path pathLockFile = directory / lockfile_name; // If a lock for this directory already exists in the map, don't try to re-lock it @@ -110,13 +111,13 @@ bool LockDirectory(const fs::path& directory, const std::string lockfile_name, b void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name) { - std::lock_guard<std::mutex> lock(cs_dir_locks); + LOCK(cs_dir_locks); dir_locks.erase((directory / lockfile_name).string()); } void ReleaseDirectoryLocks() { - std::lock_guard<std::mutex> ulock(cs_dir_locks); + LOCK(cs_dir_locks); dir_locks.clear(); } @@ -1019,7 +1020,7 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) { } ftruncate(fileno(file), static_cast<off_t>(offset) + length); #else - #if defined(__linux__) + #if defined(HAVE_POSIX_FALLOCATE) // Version using posix_fallocate off_t nEndPos = (off_t)offset + length; if (0 == posix_fallocate(fileno(file), 0, nEndPos)) return; diff --git a/src/util/translation.h b/src/util/translation.h index fc45da440a..268bcf30a7 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -16,8 +16,24 @@ struct bilingual_str { std::string original; std::string translated; + + bilingual_str& operator+=(const bilingual_str& rhs) + { + original += rhs.original; + translated += rhs.translated; + return *this; + } }; +inline bilingual_str operator+(bilingual_str lhs, const bilingual_str& rhs) +{ + lhs += rhs; + return lhs; +} + +/** Mark a bilingual_str as untranslated */ +inline bilingual_str Untranslated(std::string original) { return {original, original}; } + namespace tinyformat { template <typename... Args> bilingual_str format(const bilingual_str& fmt, const Args&... args) diff --git a/src/validation.cpp b/src/validation.cpp index d18803b342..dbdf5028fd 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -196,8 +196,8 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc std::unique_ptr<CBlockTreeDB> pblocktree; // See definition for documentation -static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); -static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); +static void FindFilesToPruneManual(ChainstateManager& chainman, std::set<int>& setFilesToPrune, int nManualPruneHeight); +static void FindFilesToPrune(ChainstateManager& chainman, std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); @@ -1651,14 +1651,15 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) } /** Abort with a message */ +// TODO: AbortNode() should take bilingual_str userMessage parameter. static bool AbortNode(const std::string& strMessage, const std::string& userMessage = "", unsigned int prefix = 0) { SetMiscWarning(strMessage); LogPrintf("*** %s\n", strMessage); if (!userMessage.empty()) { - uiInterface.ThreadSafeMessageBox(userMessage, "", CClientUIInterface::MSG_ERROR | prefix); + uiInterface.ThreadSafeMessageBox(Untranslated(userMessage), "", CClientUIInterface::MSG_ERROR | prefix); } else { - uiInterface.ThreadSafeMessageBox(_("Error: A fatal internal error occurred, see debug.log for details").translated, "", CClientUIInterface::MSG_ERROR | CClientUIInterface::MSG_NOPREFIX); + uiInterface.ThreadSafeMessageBox(_("Error: A fatal internal error occurred, see debug.log for details"), "", CClientUIInterface::MSG_ERROR | CClientUIInterface::MSG_NOPREFIX); } StartShutdown(); return false; @@ -2279,13 +2280,13 @@ bool CChainState::FlushStateToDisk( LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { if (nManualPruneHeight > 0) { - LOG_TIME_MILLIS("find files to prune (manual)", BCLog::BENCH); + LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCH); - FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight); + FindFilesToPruneManual(g_chainman, setFilesToPrune, nManualPruneHeight); } else { - LOG_TIME_MILLIS("find files to prune", BCLog::BENCH); + LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); - FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight()); + FindFilesToPrune(g_chainman, setFilesToPrune, chainparams.PruneAfterHeight()); fCheckForPruning = false; } if (!setFilesToPrune.empty()) { @@ -2321,7 +2322,7 @@ bool CChainState::FlushStateToDisk( return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); } { - LOG_TIME_MILLIS("write block and undo data to disk", BCLog::BENCH); + LOG_TIME_MILLIS_WITH_CATEGORY("write block and undo data to disk", BCLog::BENCH); // First make sure all block and undo data is flushed to disk. FlushBlockFile(); @@ -2329,7 +2330,7 @@ bool CChainState::FlushStateToDisk( // Then update all block file information (which may refer to block and undo files). { - LOG_TIME_MILLIS("write block index to disk", BCLog::BENCH); + LOG_TIME_MILLIS_WITH_CATEGORY("write block index to disk", BCLog::BENCH); std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; vFiles.reserve(setDirtyFileInfo.size()); @@ -2349,7 +2350,7 @@ bool CChainState::FlushStateToDisk( } // Finally remove any pruned files if (fFlushForPrune) { - LOG_TIME_MILLIS("unlink pruned files", BCLog::BENCH); + LOG_TIME_MILLIS_WITH_CATEGORY("unlink pruned files", BCLog::BENCH); UnlinkPrunedFiles(setFilesToPrune); } @@ -2799,6 +2800,13 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai return true; } +static SynchronizationState GetSynchronizationState(bool init) +{ + if (!init) return SynchronizationState::POST_INIT; + if (::fReindex) return SynchronizationState::INIT_REINDEX; + return SynchronizationState::INIT_DOWNLOAD; +} + static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { bool fNotify = false; bool fInitialBlockDownload = false; @@ -2816,7 +2824,7 @@ static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { } // Send block tip changed notifications without cs_main if (fNotify) { - uiInterface.NotifyHeaderTip(fInitialBlockDownload, pindexHeader); + uiInterface.NotifyHeaderTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader); } return fNotify; } @@ -2905,7 +2913,7 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); // Always notify the UI if a new block tip was connected - uiInterface.NotifyBlockTip(fInitialDownload, pindexNewTip); + uiInterface.NotifyBlockTip(GetSynchronizationState(fInitialDownload), pindexNewTip); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). @@ -3096,7 +3104,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParam // Only notify about a new block tip if the active chain was modified. if (pindex_was_in_chain) { - uiInterface.NotifyBlockTip(IsInitialBlockDownload(), to_mark_failed->pprev); + uiInterface.NotifyBlockTip(GetSynchronizationState(IsInitialBlockDownload()), to_mark_failed->pprev); } return true; } @@ -3386,7 +3394,14 @@ int GetWitnessCommitmentIndex(const CBlock& block) int commitpos = -1; if (!block.vtx.empty()) { for (size_t o = 0; o < block.vtx[0]->vout.size(); o++) { - if (block.vtx[0]->vout[o].scriptPubKey.size() >= 38 && block.vtx[0]->vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0]->vout[o].scriptPubKey[1] == 0x24 && block.vtx[0]->vout[o].scriptPubKey[2] == 0xaa && block.vtx[0]->vout[o].scriptPubKey[3] == 0x21 && block.vtx[0]->vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0]->vout[o].scriptPubKey[5] == 0xed) { + const CTxOut& vout = block.vtx[0]->vout[o]; + if (vout.scriptPubKey.size() >= MINIMUM_WITNESS_COMMITMENT && + vout.scriptPubKey[0] == OP_RETURN && + vout.scriptPubKey[1] == 0x24 && + vout.scriptPubKey[2] == 0xaa && + vout.scriptPubKey[3] == 0x21 && + vout.scriptPubKey[4] == 0xa9 && + vout.scriptPubKey[5] == 0xed) { commitpos = o; } } @@ -3417,7 +3432,7 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc CHash256().Write(witnessroot.begin(), 32).Write(ret.data(), 32).Finalize(witnessroot.begin()); CTxOut out; out.nValue = 0; - out.scriptPubKey.resize(38); + out.scriptPubKey.resize(MINIMUM_WITNESS_COMMITMENT); out.scriptPubKey[0] = OP_RETURN; out.scriptPubKey[1] = 0x24; out.scriptPubKey[2] = 0xaa; @@ -3676,13 +3691,14 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS } // Exposed wrapper for AcceptBlockHeader -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) +bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) { + AssertLockNotHeld(cs_main); { LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast - bool accepted = g_chainman.m_blockman.AcceptBlockHeader( + bool accepted = m_blockman.AcceptBlockHeader( header, state, chainparams, &pindex); ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); @@ -3804,7 +3820,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, Block return true; } -bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool *fNewBlock) +bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) { AssertLockNotHeld(cs_main); @@ -3880,12 +3896,12 @@ uint64_t CalculateCurrentUsage() return retval; } -/* Prune a block file (modify associated database entries)*/ -void PruneOneBlockFile(const int fileNumber) +void ChainstateManager::PruneOneBlockFile(const int fileNumber) { + AssertLockHeld(cs_main); LOCK(cs_LastBlockFile); - for (const auto& entry : g_chainman.BlockIndex()) { + for (const auto& entry : m_blockman.m_block_index) { CBlockIndex* pindex = entry.second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; @@ -3899,12 +3915,12 @@ void PruneOneBlockFile(const int fileNumber) // to be downloaded again in order to consider its chain, at which // point it would be considered as a candidate for // m_blocks_unlinked or setBlockIndexCandidates. - auto range = g_chainman.m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); + auto range = m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); while (range.first != range.second) { std::multimap<CBlockIndex *, CBlockIndex *>::iterator _it = range.first; range.first++; if (_it->second == pindex) { - g_chainman.m_blockman.m_blocks_unlinked.erase(_it); + m_blockman.m_blocks_unlinked.erase(_it); } } } @@ -3926,7 +3942,7 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) } /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ -static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight) +static void FindFilesToPruneManual(ChainstateManager& chainman, std::set<int>& setFilesToPrune, int nManualPruneHeight) { assert(fPruneMode && nManualPruneHeight > 0); @@ -3940,7 +3956,7 @@ static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPr for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) continue; - PruneOneBlockFile(fileNumber); + chainman.PruneOneBlockFile(fileNumber); setFilesToPrune.insert(fileNumber); count++; } @@ -3973,7 +3989,7 @@ void PruneBlockFilesManual(int nManualPruneHeight) * * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned */ -static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight) +static void FindFilesToPrune(ChainstateManager& chainman, std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight) { LOCK2(cs_main, cs_LastBlockFile); if (::ChainActive().Tip() == nullptr || nPruneTarget == 0) { @@ -4015,7 +4031,7 @@ static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfte if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) continue; - PruneOneBlockFile(fileNumber); + chainman.PruneOneBlockFile(fileNumber); // Queue up the files for removal setFilesToPrune.insert(fileNumber); nCurrentUsage -= nBytesToPrune; @@ -4139,9 +4155,9 @@ void BlockManager::Unload() { m_block_index.clear(); } -bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool static LoadBlockIndexDB(ChainstateManager& chainman, const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - if (!g_chainman.m_blockman.LoadBlockIndex( + if (!chainman.m_blockman.LoadBlockIndex( chainparams.GetConsensus(), *pblocktree, ::ChainstateActive().setBlockIndexCandidates)) { return false; @@ -4167,8 +4183,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - for (const std::pair<const uint256, CBlockIndex*>& item : g_chainman.BlockIndex()) - { + for (const std::pair<const uint256, CBlockIndex*>& item : chainman.BlockIndex()) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { setBlkDataFiles.insert(pindex->nFile); @@ -4585,14 +4600,15 @@ void UnloadBlockIndex() fHavePruned = false; } -bool LoadBlockIndex(const CChainParams& chainparams) +bool ChainstateManager::LoadBlockIndex(const CChainParams& chainparams) { + AssertLockHeld(cs_main); // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret = LoadBlockIndexDB(chainparams); + bool ret = LoadBlockIndexDB(*this, chainparams); if (!ret) return false; - needs_init = g_chainman.m_blockman.m_block_index.empty(); + needs_init = m_blockman.m_block_index.empty(); } if (needs_init) { @@ -4637,7 +4653,7 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return ::ChainstateActive().LoadGenesisBlock(chainparams); } -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp) +void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; @@ -4649,7 +4665,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SERIALIZED_SIZE, MAX_BLOCK_SERIALIZED_SIZE+8, SER_DISK, CLIENT_VERSION); uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { - boost::this_thread::interruption_point(); + if (ShutdownRequested()) return; blkdat.SetPos(nRewind); nRewind++; // start one byte further next time, in case of failure @@ -4754,9 +4770,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi } catch (const std::runtime_error& e) { AbortNode(std::string("System error: ") + e.what()); } - if (nLoaded > 0) - LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, GetTimeMillis() - nStart); - return nLoaded > 0; + LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, GetTimeMillis() - nStart); } void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) diff --git a/src/validation.h b/src/validation.h index 91b1ba6497..8112e38704 100644 --- a/src/validation.h +++ b/src/validation.h @@ -43,6 +43,7 @@ class CConnman; class CScriptCheck; class CBlockPolicyEstimator; class CTxMemPool; +class ChainstateManager; class TxValidationState; struct ChainTxData; @@ -92,6 +93,8 @@ static const unsigned int DEFAULT_CHECKLEVEL = 3; // one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB // Setting the target to >= 550 MiB will make it likely we can respect the target. static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; +/** Minimum size of a witness commitment structure. Defined in BIP 141. **/ +static constexpr size_t MINIMUM_WITNESS_COMMITMENT{38}; struct BlockHasher { @@ -101,6 +104,13 @@ struct BlockHasher size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } }; +/** Current sync state passed to tip changed callbacks. */ +enum class SynchronizationState { + INIT_REINDEX, + INIT_DOWNLOAD, + POST_INIT +}; + extern RecursiveMutex cs_main; extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; @@ -140,52 +150,14 @@ extern bool fPruneMode; /** Number of MiB of block files that we're trying to stay below. */ extern uint64_t nPruneTarget; -/** - * Process an incoming block. This only returns after the best known valid - * block is made active. Note that it does not, however, guarantee that the - * specific block passed to it has been checked for validity! - * - * If you want to *possibly* get feedback on whether pblock is valid, you must - * install a CValidationInterface (see validationinterface.h) - this will have - * its BlockChecked method called whenever *any* block completes validation. - * - * Note that we guarantee that either the proof-of-work is valid on pblock, or - * (and possibly also) BlockChecked will have been called. - * - * May not be called in a - * validationinterface callback. - * - * @param[in] pblock The block we want to process. - * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. - * @param[out] fNewBlock A boolean which is set to indicate if the block was first received via this call - * @returns If the block was processed, independently of block validity - */ -bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) LOCKS_EXCLUDED(cs_main); - -/** - * Process incoming block headers. - * - * May not be called in a - * validationinterface callback. - * - * @param[in] block The block headers themselves - * @param[out] state This may be set to an Error state if any error occurred processing them - * @param[in] chainparams The params for the chain we want to connect to - * @param[out] ppindex If set, the pointer will be set to point to the last new block index object for the given headers - */ -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); - /** Open a block file (blk?????.dat) */ FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ fs::path GetBlockPosFilename(const FlatFilePos &pos); /** Import blocks from an external file */ -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp = nullptr); +void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); -/** Load the block tree and coins database from disk, - * initializing state if we're running with -reindex. */ -bool LoadBlockIndex(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Unload database information */ void UnloadBlockIndex(); /** Run an instance of the script checking thread */ @@ -208,11 +180,6 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pin uint64_t CalculateCurrentUsage(); /** - * Mark one block file as pruned. - */ -void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - -/** * Actually unlink the specified files */ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); @@ -484,9 +451,6 @@ enum class CoinsCacheSizeState OK = 0 }; -// Defined below, but needed for `friend` usage in CChainState. -class ChainstateManager; - /** * CChainState stores and provides an API to update our local knowledge of the * current best chain. @@ -861,6 +825,47 @@ public: CChain& ValidatedChain() const { return ValidatedChainstate().m_chain; } CBlockIndex* ValidatedTip() const { return ValidatedChain().Tip(); } + /** + * Process an incoming block. This only returns after the best known valid + * block is made active. Note that it does not, however, guarantee that the + * specific block passed to it has been checked for validity! + * + * If you want to *possibly* get feedback on whether pblock is valid, you must + * install a CValidationInterface (see validationinterface.h) - this will have + * its BlockChecked method called whenever *any* block completes validation. + * + * Note that we guarantee that either the proof-of-work is valid on pblock, or + * (and possibly also) BlockChecked will have been called. + * + * May not be called in a + * validationinterface callback. + * + * @param[in] pblock The block we want to process. + * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. + * @param[out] fNewBlock A boolean which is set to indicate if the block was first received via this call + * @returns If the block was processed, independently of block validity + */ + bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) LOCKS_EXCLUDED(cs_main); + + /** + * Process incoming block headers. + * + * May not be called in a + * validationinterface callback. + * + * @param[in] block The block headers themselves + * @param[out] state This may be set to an Error state if any error occurred processing them + * @param[in] chainparams The params for the chain we want to connect to + * @param[out] ppindex If set, the pointer will be set to point to the last new block index object for the given headers + */ + bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); + + //! Mark one block file as pruned (modify associated database entries) + void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Load the block tree and coins database from disk, initializing state if we're running with -reindex + bool LoadBlockIndex(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + //! Unload block index and chain data before shutdown. void Unload() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); @@ -868,6 +873,7 @@ public: void Reset(); }; +/** DEPRECATED! Please use node.chainman instead. May only be used in validation.cpp internally */ extern ChainstateManager g_chainman GUARDED_BY(::cs_main); /** @returns the most-work valid chainstate. */ diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 11000774c0..9437f9c817 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -89,22 +89,26 @@ public: static CMainSignals g_signals; -void CMainSignals::RegisterBackgroundSignalScheduler(CScheduler& scheduler) { +void CMainSignals::RegisterBackgroundSignalScheduler(CScheduler& scheduler) +{ assert(!m_internals); m_internals.reset(new MainSignalsInstance(&scheduler)); } -void CMainSignals::UnregisterBackgroundSignalScheduler() { +void CMainSignals::UnregisterBackgroundSignalScheduler() +{ m_internals.reset(nullptr); } -void CMainSignals::FlushBackgroundCallbacks() { +void CMainSignals::FlushBackgroundCallbacks() +{ if (m_internals) { m_internals->m_schedulerClient.EmptyQueue(); } } -size_t CMainSignals::CallbacksPending() { +size_t CMainSignals::CallbacksPending() +{ if (!m_internals) return 0; return m_internals->m_schedulerClient.CallbacksPending(); } @@ -114,10 +118,11 @@ CMainSignals& GetMainSignals() return g_signals; } -void RegisterSharedValidationInterface(std::shared_ptr<CValidationInterface> pwalletIn) { - // Each connection captures pwalletIn to ensure that each callback is - // executed before pwalletIn is destroyed. For more details see #18338. - g_signals.m_internals->Register(std::move(pwalletIn)); +void RegisterSharedValidationInterface(std::shared_ptr<CValidationInterface> callbacks) +{ + // Each connection captures the shared_ptr to ensure that each callback is + // executed before the subscriber is destroyed. For more details see #18338. + g_signals.m_internals->Register(std::move(callbacks)); } void RegisterValidationInterface(CValidationInterface* callbacks) @@ -132,24 +137,28 @@ void UnregisterSharedValidationInterface(std::shared_ptr<CValidationInterface> c UnregisterValidationInterface(callbacks.get()); } -void UnregisterValidationInterface(CValidationInterface* pwalletIn) { +void UnregisterValidationInterface(CValidationInterface* callbacks) +{ if (g_signals.m_internals) { - g_signals.m_internals->Unregister(pwalletIn); + g_signals.m_internals->Unregister(callbacks); } } -void UnregisterAllValidationInterfaces() { +void UnregisterAllValidationInterfaces() +{ if (!g_signals.m_internals) { return; } g_signals.m_internals->Clear(); } -void CallFunctionInValidationInterfaceQueue(std::function<void ()> func) { +void CallFunctionInValidationInterfaceQueue(std::function<void()> func) +{ g_signals.m_internals->m_schedulerClient.AddToProcessQueue(std::move(func)); } -void SyncWithValidationInterfaceQueue() { +void SyncWithValidationInterfaceQueue() +{ AssertLockNotHeld(cs_main); // Block until the validation queue drains std::promise<void> promise; diff --git a/src/validationinterface.h b/src/validationinterface.h index cb0204a555..9c23965bc1 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -22,20 +22,20 @@ class CValidationInterface; class uint256; class CScheduler; -// These functions dispatch to one or all registered wallets - -/** Register a wallet to receive updates from core */ -void RegisterValidationInterface(CValidationInterface* pwalletIn); -/** Unregister a wallet from core */ -void UnregisterValidationInterface(CValidationInterface* pwalletIn); -/** Unregister all wallets from core */ +/** Register subscriber */ +void RegisterValidationInterface(CValidationInterface* callbacks); +/** Unregister subscriber. DEPRECATED. This is not safe to use when the RPC server or main message handler thread is running. */ +void UnregisterValidationInterface(CValidationInterface* callbacks); +/** Unregister all subscribers */ void UnregisterAllValidationInterfaces(); // Alternate registration functions that release a shared_ptr after the last // notification is sent. These are useful for race-free cleanup, since // unregistration is nonblocking and can return before the last notification is // processed. +/** Register subscriber */ void RegisterSharedValidationInterface(std::shared_ptr<CValidationInterface> callbacks); +/** Unregister subscriber */ void UnregisterSharedValidationInterface(std::shared_ptr<CValidationInterface> callbacks); /** diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index f59c63260e..f2df786e2e 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -43,15 +43,9 @@ public: //! such as the various parameters to scrypt std::vector<unsigned char> vchOtherDerivationParameters; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vchCryptedKey); - READWRITE(vchSalt); - READWRITE(nDerivationMethod); - READWRITE(nDeriveIterations); - READWRITE(vchOtherDerivationParameters); + SERIALIZE_METHODS(CMasterKey, obj) + { + READWRITE(obj.vchCryptedKey, obj.vchSalt, obj.nDerivationMethod, obj.nDeriveIterations, obj.vchOtherDerivationParameters); } CMasterKey() diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 67be4d85d2..4ed28b0623 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -268,21 +268,14 @@ BerkeleyEnvironment::BerkeleyEnvironment() fMockDb = true; } -BerkeleyEnvironment::VerifyResult BerkeleyEnvironment::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename) +bool BerkeleyEnvironment::Verify(const std::string& strFile) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); - if (result == 0) - return VerifyResult::VERIFY_OK; - else if (recoverFunc == nullptr) - return VerifyResult::RECOVER_FAIL; - - // Try to recover: - bool fRecovered = (*recoverFunc)(fs::path(strPath) / strFile, out_backup_filename); - return (fRecovered ? VerifyResult::RECOVER_OK : VerifyResult::RECOVER_FAIL); + return result == 0; } BerkeleyBatch::SafeDbt::SafeDbt() @@ -324,93 +317,24 @@ BerkeleyBatch::SafeDbt::operator Dbt*() return &m_dbt; } -bool BerkeleyBatch::Recover(const fs::path& file_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename) -{ - std::string filename; - std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); - - // Recovery procedure: - // move wallet file to walletfilename.timestamp.bak - // Call Salvage with fAggressive=true to - // get as much data as possible. - // Rewrite salvaged data to fresh wallet file - // Set -rescan so any missing transactions will be - // found. - int64_t now = GetTime(); - newFilename = strprintf("%s.%d.bak", filename, now); - - int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr, - newFilename.c_str(), DB_AUTO_COMMIT); - if (result == 0) - LogPrintf("Renamed %s to %s\n", filename, newFilename); - else - { - LogPrintf("Failed to rename %s to %s\n", filename, newFilename); - return false; - } - - std::vector<BerkeleyEnvironment::KeyValPair> salvagedData; - bool fSuccess = env->Salvage(newFilename, true, salvagedData); - if (salvagedData.empty()) - { - LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); - return false; - } - LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); - - std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); - int ret = pdbCopy->open(nullptr, // Txn pointer - filename.c_str(), // Filename - "main", // Logical db name - DB_BTREE, // Database type - DB_CREATE, // Flags - 0); - if (ret > 0) { - LogPrintf("Cannot create database file %s\n", filename); - pdbCopy->close(0); - return false; - } - - DbTxn* ptxn = env->TxnBegin(); - for (BerkeleyEnvironment::KeyValPair& row : salvagedData) - { - if (recoverKVcallback) - { - CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); - CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); - if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue)) - continue; - } - Dbt datKey(&row.first[0], row.first.size()); - Dbt datValue(&row.second[0], row.second.size()); - int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); - if (ret2 > 0) - fSuccess = false; - } - ptxn->commit(0); - pdbCopy->close(0); - - return fSuccess; -} - -bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, std::string& errorStr) +bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr) { std::string walletFile; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); fs::path walletDir = env->Directory(); - LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(nullptr, nullptr, nullptr)); + LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); LogPrintf("Using wallet %s\n", file_path.string()); if (!env->Open(true /* retry */)) { - errorStr = strprintf(_("Error initializing wallet database environment %s!").translated, walletDir); + errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir); return false; } return true; } -bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::vector<std::string>& warnings, std::string& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc) +bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, bilingual_str& errorStr) { std::string walletFile; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile); @@ -418,19 +342,8 @@ bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::vector<st if (fs::exists(walletDir / walletFile)) { - std::string backup_filename; - BerkeleyEnvironment::VerifyResult r = env->Verify(walletFile, recoverFunc, backup_filename); - if (r == BerkeleyEnvironment::VerifyResult::RECOVER_OK) - { - warnings.push_back(strprintf(_("Warning: Wallet file corrupt, data salvaged!" - " Original %s saved as %s in %s; if" - " your balance or transactions are incorrect you should" - " restore from a backup.").translated, - walletFile, backup_filename, walletDir)); - } - if (r == BerkeleyEnvironment::VerifyResult::RECOVER_FAIL) - { - errorStr = strprintf(_("%s corrupt, salvage failed").translated, walletFile); + if (!env->Verify(walletFile)) { + errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), walletFile); return false; } } @@ -438,72 +351,6 @@ bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::vector<st return true; } -/* End of headers, beginning of key/value data */ -static const char *HEADER_END = "HEADER=END"; -/* End of key/value data */ -static const char *DATA_END = "DATA=END"; - -bool BerkeleyEnvironment::Salvage(const std::string& strFile, bool fAggressive, std::vector<BerkeleyEnvironment::KeyValPair>& vResult) -{ - LOCK(cs_db); - assert(mapFileUseCount.count(strFile) == 0); - - u_int32_t flags = DB_SALVAGE; - if (fAggressive) - flags |= DB_AGGRESSIVE; - - std::stringstream strDump; - - Db db(dbenv.get(), 0); - int result = db.verify(strFile.c_str(), nullptr, &strDump, flags); - if (result == DB_VERIFY_BAD) { - LogPrintf("BerkeleyEnvironment::Salvage: Database salvage found errors, all data may not be recoverable.\n"); - if (!fAggressive) { - LogPrintf("BerkeleyEnvironment::Salvage: Rerun with aggressive mode to ignore errors and continue.\n"); - return false; - } - } - if (result != 0 && result != DB_VERIFY_BAD) { - LogPrintf("BerkeleyEnvironment::Salvage: Database salvage failed with result %d.\n", result); - return false; - } - - // Format of bdb dump is ascii lines: - // header lines... - // HEADER=END - // hexadecimal key - // hexadecimal value - // ... repeated - // DATA=END - - std::string strLine; - while (!strDump.eof() && strLine != HEADER_END) - getline(strDump, strLine); // Skip past header - - std::string keyHex, valueHex; - while (!strDump.eof() && keyHex != DATA_END) { - getline(strDump, keyHex); - if (keyHex != DATA_END) { - if (strDump.eof()) - break; - getline(strDump, valueHex); - if (valueHex == DATA_END) { - LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Number of keys in data does not match number of values.\n"); - break; - } - vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex))); - } - } - - if (keyHex != DATA_END) { - LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Unexpected end of file while reading salvage output.\n"); - return false; - } - - return (result == 0); -} - - void BerkeleyEnvironment::CheckpointLSN(const std::string& strFile) { dbenv->txn_checkpoint(0, 0, 0); @@ -916,3 +763,8 @@ void BerkeleyDatabase::ReloadDbEnv() env->ReloadDbEnv(); } } + +std::string BerkeleyDatabaseVersion() +{ + return DbEnv::version(nullptr, nullptr, nullptr); +} diff --git a/src/wallet/db.h b/src/wallet/db.h index 1c5f42abca..54ce144ffc 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -19,7 +19,16 @@ #include <unordered_map> #include <vector> +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsuggest-override" +#endif #include <db_cxx.h> +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + +struct bilingual_str; static const unsigned int DEFAULT_WALLET_DBLOGSIZE = 100; static const bool DEFAULT_WALLET_PRIVDB = true; @@ -57,26 +66,7 @@ public: bool IsDatabaseLoaded(const std::string& db_filename) const { return m_databases.find(db_filename) != m_databases.end(); } fs::path Directory() const { return strPath; } - /** - * Verify that database file strFile is OK. If it is not, - * call the callback to try to recover. - * This must be called BEFORE strFile is opened. - * Returns true if strFile is OK. - */ - enum class VerifyResult { VERIFY_OK, - RECOVER_OK, - RECOVER_FAIL }; - typedef bool (*recoverFunc_type)(const fs::path& file_path, std::string& out_backup_filename); - VerifyResult Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename); - /** - * Salvage data from a file that Verify says is bad. - * fAggressive sets the DB_AGGRESSIVE flag (see berkeley DB->verify() method documentation). - * Appends binary key/value pairs to vResult, returns true if successful. - * NOTE: reads the entire database into memory, so cannot be used - * for huge databases. - */ - typedef std::pair<std::vector<unsigned char>, std::vector<unsigned char> > KeyValPair; - bool Salvage(const std::string& strFile, bool fAggressive, std::vector<KeyValPair>& vResult); + bool Verify(const std::string& strFile); bool Open(bool retry); void Close(); @@ -236,15 +226,14 @@ public: void Flush(); void Close(); - static bool Recover(const fs::path& file_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename); /* flush the wallet passively (TRY_LOCK) ideal to be called periodically */ static bool PeriodicFlush(BerkeleyDatabase& database); /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path& file_path, std::string& errorStr); + static bool VerifyEnvironment(const fs::path& file_path, bilingual_str& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& file_path, std::vector<std::string>& warnings, std::string& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc); + static bool VerifyDatabaseFile(const fs::path& file_path, bilingual_str& errorStr); template <typename K, typename T> bool Read(const K& key, T& value) @@ -400,4 +389,6 @@ public: bool static Rewrite(BerkeleyDatabase& database, const char* pszSkip = nullptr); }; +std::string BerkeleyDatabaseVersion(); + #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index dafa022ded..cacf306891 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -3,44 +3,45 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <interfaces/chain.h> -#include <wallet/coincontrol.h> -#include <wallet/feebumper.h> -#include <wallet/fees.h> -#include <wallet/wallet.h> #include <policy/fees.h> #include <policy/policy.h> #include <util/moneystr.h> #include <util/rbf.h> #include <util/system.h> +#include <util/translation.h> +#include <wallet/coincontrol.h> +#include <wallet/feebumper.h> +#include <wallet/fees.h> +#include <wallet/wallet.h> //! Check whether transaction has descendant in wallet or mempool, or has been //! mined, or conflicts with a mined transaction. Return a feebumper::Result. -static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWalletTx& wtx, std::vector<std::string>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWalletTx& wtx, std::vector<bilingual_str>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { if (wallet.HasWalletSpend(wtx.GetHash())) { - errors.push_back("Transaction has descendants in the wallet"); + errors.push_back(Untranslated("Transaction has descendants in the wallet")); return feebumper::Result::INVALID_PARAMETER; } { if (wallet.chain().hasDescendantsInMempool(wtx.GetHash())) { - errors.push_back("Transaction has descendants in the mempool"); + errors.push_back(Untranslated("Transaction has descendants in the mempool")); return feebumper::Result::INVALID_PARAMETER; } } if (wtx.GetDepthInMainChain() != 0) { - errors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); + errors.push_back(Untranslated("Transaction has been mined, or is conflicted with a mined transaction")); return feebumper::Result::WALLET_ERROR; } if (!SignalsOptInRBF(*wtx.tx)) { - errors.push_back("Transaction is not BIP 125 replaceable"); + errors.push_back(Untranslated("Transaction is not BIP 125 replaceable")); return feebumper::Result::WALLET_ERROR; } if (wtx.mapValue.count("replaced_by_txid")) { - errors.push_back(strprintf("Cannot bump transaction %s which was already bumped by transaction %s", wtx.GetHash().ToString(), wtx.mapValue.at("replaced_by_txid"))); + errors.push_back(strprintf(Untranslated("Cannot bump transaction %s which was already bumped by transaction %s"), wtx.GetHash().ToString(), wtx.mapValue.at("replaced_by_txid"))); return feebumper::Result::WALLET_ERROR; } @@ -48,7 +49,7 @@ static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWallet // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; if (!wallet.IsAllFromMe(*wtx.tx, filter)) { - errors.push_back("Transaction contains inputs that don't belong to this wallet"); + errors.push_back(Untranslated("Transaction contains inputs that don't belong to this wallet")); return feebumper::Result::WALLET_ERROR; } @@ -57,7 +58,8 @@ static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWallet } //! Check if the user provided a valid feeRate -static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wtx, const CFeeRate& newFeerate, const int64_t maxTxSize, std::vector<std::string>& errors) { +static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wtx, const CFeeRate& newFeerate, const int64_t maxTxSize, std::vector<bilingual_str>& errors) +{ // check that fee rate is higher than mempool's minimum fee // (no point in bumping fee if we know that the new tx won't be accepted to the mempool) // This may occur if the user set fee_rate or paytxfee too low, if fallbackfee is too low, or, perhaps, @@ -67,7 +69,7 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt if (newFeerate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { errors.push_back(strprintf( - "New fee rate (%s) is lower than the minimum fee rate (%s) to get into the mempool -- ", + Untranslated("New fee rate (%s) is lower than the minimum fee rate (%s) to get into the mempool -- "), FormatMoney(newFeerate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()))); return feebumper::Result::WALLET_ERROR; @@ -86,14 +88,14 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt CAmount minTotalFee = nOldFeeRate.GetFee(maxTxSize) + incrementalRelayFee.GetFee(maxTxSize); if (new_total_fee < minTotalFee) { - errors.push_back(strprintf("Insufficient total fee %s, must be at least %s (oldFee %s + incrementalFee %s)", + errors.push_back(strprintf(Untranslated("Insufficient total fee %s, must be at least %s (oldFee %s + incrementalFee %s)"), FormatMoney(new_total_fee), FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxTxSize)), FormatMoney(incrementalRelayFee.GetFee(maxTxSize)))); return feebumper::Result::INVALID_PARAMETER; } CAmount requiredFee = GetRequiredFee(wallet, maxTxSize); if (new_total_fee < requiredFee) { - errors.push_back(strprintf("Insufficient total fee (cannot be less than required fee %s)", + errors.push_back(strprintf(Untranslated("Insufficient total fee (cannot be less than required fee %s)"), FormatMoney(requiredFee))); return feebumper::Result::INVALID_PARAMETER; } @@ -101,8 +103,8 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt // Check that in all cases the new fee doesn't violate maxTxFee const CAmount max_tx_fee = wallet.m_default_max_tx_fee; if (new_total_fee > max_tx_fee) { - errors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than -maxtxfee %s)", - FormatMoney(new_total_fee), FormatMoney(max_tx_fee))); + errors.push_back(strprintf(Untranslated("Specified or calculated fee %s is too high (cannot be higher than -maxtxfee %s)"), + FormatMoney(new_total_fee), FormatMoney(max_tx_fee))); return feebumper::Result::WALLET_ERROR; } @@ -140,28 +142,26 @@ namespace feebumper { bool TransactionCanBeBumped(const CWallet& wallet, const uint256& txid) { - auto locked_chain = wallet.chain().lock(); LOCK(wallet.cs_wallet); const CWalletTx* wtx = wallet.GetWalletTx(txid); if (wtx == nullptr) return false; - std::vector<std::string> errors_dummy; + std::vector<bilingual_str> errors_dummy; feebumper::Result res = PreconditionChecks(wallet, *wtx, errors_dummy); return res == feebumper::Result::OK; } -Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCoinControl& coin_control, std::vector<std::string>& errors, +Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCoinControl& coin_control, std::vector<bilingual_str>& errors, CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) { // We are going to modify coin control later, copy to re-use CCoinControl new_coin_control(coin_control); - auto locked_chain = wallet.chain().lock(); LOCK(wallet.cs_wallet); errors.clear(); auto it = wallet.mapWallet.find(txid); if (it == wallet.mapWallet.end()) { - errors.push_back("Invalid or non-wallet transaction id"); + errors.push_back(Untranslated("Invalid or non-wallet transaction id")); return Result::INVALID_ADDRESS_OR_KEY; } const CWalletTx& wtx = it->second; @@ -218,9 +218,9 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo CTransactionRef tx_new = MakeTransactionRef(); CAmount fee_ret; int change_pos_in_out = -1; // No requested location for change - std::string fail_reason; - if (!wallet.CreateTransaction(*locked_chain, recipients, tx_new, fee_ret, change_pos_in_out, fail_reason, new_coin_control, false)) { - errors.push_back("Unable to create transaction: " + fail_reason); + bilingual_str fail_reason; + if (!wallet.CreateTransaction(recipients, tx_new, fee_ret, change_pos_in_out, fail_reason, new_coin_control, false)) { + errors.push_back(Untranslated("Unable to create transaction.") + Untranslated(" ") + fail_reason); return Result::WALLET_ERROR; } @@ -240,21 +240,19 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } bool SignTransaction(CWallet& wallet, CMutableTransaction& mtx) { - auto locked_chain = wallet.chain().lock(); LOCK(wallet.cs_wallet); return wallet.SignTransaction(mtx); } -Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransaction&& mtx, std::vector<std::string>& errors, uint256& bumped_txid) +Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransaction&& mtx, std::vector<bilingual_str>& errors, uint256& bumped_txid) { - auto locked_chain = wallet.chain().lock(); LOCK(wallet.cs_wallet); if (!errors.empty()) { return Result::MISC_ERROR; } auto it = txid.IsNull() ? wallet.mapWallet.end() : wallet.mapWallet.find(txid); if (it == wallet.mapWallet.end()) { - errors.push_back("Invalid or non-wallet transaction id"); + errors.push_back(Untranslated("Invalid or non-wallet transaction id")); return Result::MISC_ERROR; } CWalletTx& oldWtx = it->second; @@ -279,7 +277,7 @@ Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransacti // along with an exception. It would be good to return information about // wtxBumped to the caller even if marking the original transaction // replaced does not succeed for some reason. - errors.push_back("Created new bumpfee transaction but could not mark the original transaction as replaced"); + errors.push_back(Untranslated("Created new bumpfee transaction but could not mark the original transaction as replaced")); } return Result::OK; } diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index fc038ae731..50577c9d3e 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -12,6 +12,7 @@ class CWalletTx; class uint256; class CCoinControl; enum class FeeEstimateMode; +struct bilingual_str; namespace feebumper { @@ -30,12 +31,12 @@ bool TransactionCanBeBumped(const CWallet& wallet, const uint256& txid); //! Create bumpfee transaction based on feerate estimates. Result CreateRateBumpTransaction(CWallet& wallet, - const uint256& txid, - const CCoinControl& coin_control, - std::vector<std::string>& errors, - CAmount& old_fee, - CAmount& new_fee, - CMutableTransaction& mtx); + const uint256& txid, + const CCoinControl& coin_control, + std::vector<bilingual_str>& errors, + CAmount& old_fee, + CAmount& new_fee, + CMutableTransaction& mtx); //! Sign the new transaction, //! @return false if the tx couldn't be found or if it was @@ -47,10 +48,10 @@ bool SignTransaction(CWallet& wallet, CMutableTransaction& mtx); //! but sets errors if the tx could not be added to the mempool (will try later) //! or if the old transaction could not be marked as replaced. Result CommitTransaction(CWallet& wallet, - const uint256& txid, - CMutableTransaction&& mtx, - std::vector<std::string>& errors, - uint256& bumped_txid); + const uint256& txid, + CMutableTransaction&& mtx, + std::vector<bilingual_str>& errors, + uint256& bumped_txid); } // namespace feebumper diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 8688f3df5e..3885eb6185 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -54,7 +54,6 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-paytxfee=<amt>", strprintf("Fee (in %s/kB) to add to transactions you send (default: %s)", CURRENCY_UNIT, FormatMoney(CFeeRate{DEFAULT_PAY_TX_FEE}.GetFeePerK())), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-rescan", "Rescan the block chain for missing wallet transactions on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-salvagewallet", "Attempt to recover private keys from a corrupt wallet on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-spendzeroconfchange", strprintf("Spend unconfirmed change when sending transactions (default: %u)", DEFAULT_SPEND_ZEROCONF_CHANGE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-txconfirmtarget=<n>", strprintf("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)", DEFAULT_TX_CONFIRM_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-wallet=<path>", "Specify wallet database path. Can be specified multiple times to load multiple wallets. Path is interpreted relative to <walletdir> if it is not absolute, and will be created if it does not exist (as a directory containing a wallet.dat file and log files). For backwards compatibility this will also accept names of existing data files in <walletdir>.)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); @@ -89,16 +88,6 @@ bool WalletInit::ParameterInteraction() const LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); } - if (gArgs.GetBoolArg("-salvagewallet", false)) { - if (is_multiwallet) { - return InitError(strprintf("%s is only allowed with a single wallet file", "-salvagewallet")); - } - // Rewrite just private keys: rescan to find transactions - if (gArgs.SoftSetBoolArg("-rescan", true)) { - LogPrintf("%s: parameter interaction: -salvagewallet=1 -> setting -rescan=1\n", __func__); - } - } - bool zapwallettxes = gArgs.GetBoolArg("-zapwallettxes", false); // -zapwallettxes implies dropping the mempool on startup if (zapwallettxes && gArgs.SoftSetBoolArg("-persistmempool", false)) { @@ -108,7 +97,7 @@ bool WalletInit::ParameterInteraction() const // -zapwallettxes implies a rescan if (zapwallettxes) { if (is_multiwallet) { - return InitError(strprintf("%s is only allowed with a single wallet file", "-zapwallettxes")); + return InitError(strprintf(Untranslated("%s is only allowed with a single wallet file"), "-zapwallettxes")); } if (gArgs.SoftSetBoolArg("-rescan", true)) { LogPrintf("%s: parameter interaction: -zapwallettxes enabled -> setting -rescan=1\n", __func__); @@ -116,7 +105,7 @@ bool WalletInit::ParameterInteraction() const } if (gArgs.GetBoolArg("-sysperms", false)) - return InitError("-sysperms is not allowed in combination with enabled wallet functionality"); + return InitError(Untranslated("-sysperms is not allowed in combination with enabled wallet functionality")); return true; } diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 0afd2dfcf0..8df3e78215 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -20,14 +20,14 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal // The canonical path cleans the path, preventing >1 Berkeley environment instances for the same directory fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error); if (error || !fs::exists(wallet_dir)) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist").translated, wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); return false; } else if (!fs::is_directory(wallet_dir)) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory").translated, wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); return false; // The canonical path transforms relative paths into absolute ones, so we check the non-canonical version } else if (!wallet_dir.is_absolute()) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path").translated, wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); return false; } gArgs.ForceSetArg("-walletdir", canonical_wallet_dir.string()); @@ -37,11 +37,6 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal chain.initMessage(_("Verifying wallet(s)...").translated); - // Parameter interaction code should have thrown an error if -salvagewallet - // was enabled with more than wallet file, so the wallet_files size check - // here should have no effect. - bool salvage_wallet = gArgs.GetBoolArg("-salvagewallet", false) && wallet_files.size() <= 1; - // Keep track of each wallet absolute path to detect duplicates. std::set<fs::path> wallet_paths; @@ -49,16 +44,18 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal WalletLocation location(wallet_file); if (!wallet_paths.insert(location.GetPath()).second) { - chain.initError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified.").translated, wallet_file)); + chain.initError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified."), wallet_file)); return false; } - std::string error_string; - std::vector<std::string> warnings; - bool verify_success = CWallet::Verify(chain, location, salvage_wallet, error_string, warnings); - if (!error_string.empty()) chain.initError(error_string); - if (!warnings.empty()) chain.initWarning(Join(warnings, "\n")); - if (!verify_success) return false; + bilingual_str error_string; + std::vector<bilingual_str> warnings; + bool verify_success = CWallet::Verify(chain, location, error_string, warnings); + if (!warnings.empty()) chain.initWarning(Join(warnings, Untranslated("\n"))); + if (!verify_success) { + chain.initError(error_string); + return false; + } } return true; @@ -68,10 +65,10 @@ bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& walle { try { for (const std::string& walletFile : wallet_files) { - std::string error; - std::vector<std::string> warnings; + bilingual_str error; + std::vector<bilingual_str> warnings; std::shared_ptr<CWallet> pwallet = CWallet::CreateWalletFromFile(chain, WalletLocation(walletFile), error, warnings); - if (!warnings.empty()) chain.initWarning(Join(warnings, "\n")); + if (!warnings.empty()) chain.initWarning(Join(warnings, Untranslated("\n"))); if (!pwallet) { chain.initError(error); return false; @@ -80,7 +77,7 @@ bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& walle } return true; } catch (const std::runtime_error& e) { - chain.initError(e.what()); + chain.initError(Untranslated(e.what())); return false; } } diff --git a/src/wallet/load.h b/src/wallet/load.h index 5a62e29303..e24b1f2e69 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -16,8 +16,6 @@ class Chain; } // namespace interfaces //! Responsible for reading and validating the -wallet arguments and verifying the wallet database. -//! This function will perform salvage on the wallet if requested, as long as only one wallet is -//! being loaded (WalletInit::ParameterInteraction() forbids -salvagewallet, -zapwallettxes or -upgradewallet with multiwallet). bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files); //! Load wallet databases. diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 9417e2bd58..d5f6d63a46 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -133,7 +133,6 @@ UniValue importprivkey(const JSONRPCRequest& request) WalletRescanReserver reserver(*pwallet); bool fRescan = true; { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -285,7 +284,6 @@ UniValue importaddress(const JSONRPCRequest& request) fP2SH = request.params[3].get_bool(); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); CTxDestination dest = DecodeDestination(request.params[0].get_str()); @@ -317,7 +315,6 @@ UniValue importaddress(const JSONRPCRequest& request) { RescanWallet(*pwallet, reserver); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); pwallet->ReacceptWalletTransactions(); } @@ -348,7 +345,6 @@ UniValue importprunedfunds(const JSONRPCRequest& request) if (!DecodeHexTx(tx, request.params[0].get_str())) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); uint256 hashTx = tx.GetHash(); - CWalletTx wtx(pwallet, MakeTransactionRef(std::move(tx))); CDataStream ssMB(ParseHexV(request.params[1], "proof"), SER_NETWORK, PROTOCOL_VERSION); CMerkleBlock merkleBlock; @@ -361,7 +357,6 @@ UniValue importprunedfunds(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Something wrong with merkleblock"); } - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); int height; if (!pwallet->chain().findAncestorByHash(pwallet->GetLastBlockHash(), merkleBlock.header.GetHash(), FoundBlock().height(height))) { @@ -376,10 +371,10 @@ UniValue importprunedfunds(const JSONRPCRequest& request) unsigned int txnIndex = vIndex[it - vMatch.begin()]; CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, height, merkleBlock.header.GetHash(), txnIndex); - wtx.m_confirm = confirm; - if (pwallet->IsMine(*wtx.tx)) { - pwallet->AddToWallet(wtx, false); + CTransactionRef tx_ref = MakeTransactionRef(tx); + if (pwallet->IsMine(*tx_ref)) { + pwallet->AddToWallet(std::move(tx_ref), confirm); return NullUniValue; } @@ -407,7 +402,6 @@ UniValue removeprunedfunds(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); uint256 hash(ParseHashV(request.params[0], "txid")); @@ -487,7 +481,6 @@ UniValue importpubkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey is not a valid public key"); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); std::set<CScript> script_pub_keys; @@ -505,7 +498,6 @@ UniValue importpubkey(const JSONRPCRequest& request) { RescanWallet(*pwallet, reserver); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); pwallet->ReacceptWalletTransactions(); } @@ -557,7 +549,6 @@ UniValue importwallet(const JSONRPCRequest& request) int64_t nTimeBegin = 0; bool fGood = true; { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -700,7 +691,6 @@ UniValue dumpprivkey(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); - auto locked_chain = pwallet->chain().lock(); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(pwallet); @@ -756,8 +746,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); - LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); + LOCK2(wallet.cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(&wallet); @@ -780,7 +769,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) std::map<CKeyID, int64_t> mapKeyBirth; const std::map<CKeyID, int64_t>& mapKeyPool = spk_man.GetAllReserveKeys(); - pwallet->GetKeyBirthTimes(*locked_chain, mapKeyBirth); + wallet.GetKeyBirthTimes(mapKeyBirth); std::set<CScriptID> scripts = spk_man.GetCScripts(); @@ -1379,7 +1368,6 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) int64_t nLowestTimestamp = 0; UniValue response(UniValue::VARR); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -1414,7 +1402,6 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) if (fRescan && fRunScan && requests.size()) { int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, reserver, true /* update */); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); pwallet->ReacceptWalletTransactions(); } @@ -1676,7 +1663,6 @@ UniValue importdescriptors(const JSONRPCRequest& main_request) { bool rescan = false; UniValue response(UniValue::VARR); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -1705,7 +1691,6 @@ UniValue importdescriptors(const JSONRPCRequest& main_request) { if (rescan) { int64_t scanned_time = pwallet->RescanFromTime(lowest_timestamp, reserver, true /* update */); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); pwallet->ReacceptWalletTransactions(); } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 49c583c422..2a9ac189ea 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -23,6 +23,7 @@ #include <util/moneystr.h> #include <util/string.h> #include <util/system.h> +#include <util/translation.h> #include <util/url.h> #include <util/vector.h> #include <wallet/coincontrol.h> @@ -133,7 +134,7 @@ LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_cr return *spk_man; } -static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx, UniValue& entry) +static void WalletTxToJSON(interfaces::Chain& chain, const CWalletTx& wtx, UniValue& entry) { int confirms = wtx.GetDepthInMainChain(); entry.pushKV("confirmations", confirms); @@ -148,7 +149,7 @@ static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& lo CHECK_NONFATAL(chain.findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(block_time))); entry.pushKV("blocktime", block_time); } else { - entry.pushKV("trusted", wtx.IsTrusted(locked_chain)); + entry.pushKV("trusted", wtx.IsTrusted()); } uint256 hash = wtx.GetHash(); entry.pushKV("txid", hash.GetHex()); @@ -322,7 +323,7 @@ static UniValue setlabel(const JSONRPCRequest& request) } -static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet * const pwallet, const CTxDestination &address, CAmount nValue, bool fSubtractFeeFromAmount, const CCoinControl& coin_control, mapValue_t mapValue) +static CTransactionRef SendMoney(CWallet* const pwallet, const CTxDestination& address, CAmount nValue, bool fSubtractFeeFromAmount, const CCoinControl& coin_control, mapValue_t mapValue) { CAmount curBalance = pwallet->GetBalance(0, coin_control.m_avoid_address_reuse).m_mine_trusted; @@ -338,16 +339,16 @@ static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet // Create and send the transaction CAmount nFeeRequired = 0; - std::string strError; + bilingual_str error; std::vector<CRecipient> vecSend; int nChangePosRet = -1; CRecipient recipient = {scriptPubKey, nValue, fSubtractFeeFromAmount}; vecSend.push_back(recipient); CTransactionRef tx; - if (!pwallet->CreateTransaction(locked_chain, vecSend, tx, nFeeRequired, nChangePosRet, strError, coin_control)) { + if (!pwallet->CreateTransaction(vecSend, tx, nFeeRequired, nChangePosRet, error, coin_control)) { if (!fSubtractFeeFromAmount && nValue + nFeeRequired > curBalance) - strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); - throw JSONRPCError(RPC_WALLET_ERROR, strError); + error = strprintf(Untranslated("Error: This transaction requires a transaction fee of at least %s"), FormatMoney(nFeeRequired)); + throw JSONRPCError(RPC_WALLET_ERROR, error.original); } pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */); return tx; @@ -399,7 +400,6 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); CTxDestination dest = DecodeDestination(request.params[0].get_str()); @@ -445,7 +445,7 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); - CTransactionRef tx = SendMoney(*locked_chain, pwallet, dest, nAmount, fSubtractFeeFromAmount, coin_control, std::move(mapValue)); + CTransactionRef tx = SendMoney(pwallet, dest, nAmount, fSubtractFeeFromAmount, coin_control, std::move(mapValue)); return tx->GetHash().GetHex(); } @@ -487,11 +487,10 @@ static UniValue listaddressgroupings(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); UniValue jsonGroupings(UniValue::VARR); - std::map<CTxDestination, CAmount> balances = pwallet->GetAddressBalances(*locked_chain); + std::map<CTxDestination, CAmount> balances = pwallet->GetAddressBalances(); for (const std::set<CTxDestination>& grouping : pwallet->GetAddressGroupings()) { UniValue jsonGrouping(UniValue::VARR); for (const CTxDestination& address : grouping) @@ -543,7 +542,6 @@ static UniValue signmessage(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -572,7 +570,7 @@ static UniValue signmessage(const JSONRPCRequest& request) return signature; } -static CAmount GetReceived(interfaces::Chain::Lock& locked_chain, const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { std::set<CTxDestination> address_set; @@ -602,7 +600,7 @@ static CAmount GetReceived(interfaces::Chain::Lock& locked_chain, const CWallet& CAmount amount = 0; for (const std::pair<const uint256, CWalletTx>& wtx_pair : wallet.mapWallet) { const CWalletTx& wtx = wtx_pair.second; - if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx) || wtx.GetDepthInMainChain() < min_depth) { + if (wtx.IsCoinBase() || !wallet.chain().checkFinalTx(*wtx.tx) || wtx.GetDepthInMainChain() < min_depth) { continue; } @@ -652,10 +650,9 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ false)); + return ValueFromAmount(GetReceived(*pwallet, request.params, /* by_label */ false)); } @@ -693,10 +690,9 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ true)); + return ValueFromAmount(GetReceived(*pwallet, request.params, /* by_label */ true)); } @@ -736,7 +732,6 @@ static UniValue getbalance(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); const UniValue& dummy_value = request.params[0]; @@ -778,7 +773,6 @@ static UniValue getunconfirmedbalance(const JSONRPCRequest &request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); return ValueFromAmount(pwallet->GetBalance().m_mine_untrusted_pending); @@ -841,7 +835,6 @@ static UniValue sendmany(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (!request.params[0].isNull() && !request.params[0].get_str().empty()) { @@ -911,11 +904,11 @@ static UniValue sendmany(const JSONRPCRequest& request) // Send CAmount nFeeRequired = 0; int nChangePosRet = -1; - std::string strFailReason; + bilingual_str error; CTransactionRef tx; - bool fCreated = pwallet->CreateTransaction(*locked_chain, vecSend, tx, nFeeRequired, nChangePosRet, strFailReason, coin_control); + bool fCreated = pwallet->CreateTransaction(vecSend, tx, nFeeRequired, nChangePosRet, error, coin_control); if (!fCreated) - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, error.original); pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */); return tx->GetHash().GetHex(); } @@ -963,7 +956,6 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); - auto locked_chain = pwallet->chain().lock(); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); std::string label; @@ -1016,7 +1008,7 @@ struct tallyitem } }; -static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, const CWallet* const pwallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +static UniValue ListReceived(const CWallet* const pwallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { // Minimum confirmations int nMinDepth = 1; @@ -1049,7 +1041,7 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, const CWalle for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx)) { + if (wtx.IsCoinBase() || !pwallet->chain().checkFinalTx(*wtx.tx)) { continue; } @@ -1209,10 +1201,9 @@ static UniValue listreceivedbyaddress(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - return ListReceived(*locked_chain, pwallet, request.params, false); + return ListReceived(pwallet, request.params, false); } static UniValue listreceivedbylabel(const JSONRPCRequest& request) @@ -1254,10 +1245,9 @@ static UniValue listreceivedbylabel(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - return ListReceived(*locked_chain, pwallet, request.params, true); + return ListReceived(pwallet, request.params, true); } static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) @@ -1278,7 +1268,7 @@ static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) * @param filter_ismine The "is mine" filter flags. * @param filter_label Optional label string to filter incoming transactions. */ -static void ListTransactions(interfaces::Chain::Lock& locked_chain, const CWallet* const pwallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +static void ListTransactions(const CWallet* const pwallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { CAmount nFee; std::list<COutputEntry> listReceived; @@ -1307,7 +1297,7 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, const CWalle entry.pushKV("vout", s.vout); entry.pushKV("fee", ValueFromAmount(-nFee)); if (fLong) - WalletTxToJSON(pwallet->chain(), locked_chain, wtx, entry); + WalletTxToJSON(pwallet->chain(), wtx, entry); entry.pushKV("abandoned", wtx.isAbandoned()); ret.push_back(entry); } @@ -1349,7 +1339,7 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, const CWalle } entry.pushKV("vout", r.vout); if (fLong) - WalletTxToJSON(pwallet->chain(), locked_chain, wtx, entry); + WalletTxToJSON(pwallet->chain(), wtx, entry); ret.push_back(entry); } } @@ -1464,7 +1454,6 @@ UniValue listtransactions(const JSONRPCRequest& request) UniValue ret(UniValue::VARR); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); const CWallet::TxItems & txOrdered = pwallet->wtxOrdered; @@ -1473,7 +1462,7 @@ UniValue listtransactions(const JSONRPCRequest& request) for (CWallet::TxItems::const_reverse_iterator it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) { CWalletTx *const pwtx = (*it).second; - ListTransactions(*locked_chain, pwallet, *pwtx, 0, true, ret, filter, filter_label); + ListTransactions(pwallet, *pwtx, 0, true, ret, filter, filter_label); if ((int)ret.size() >= (nCount+nFrom)) break; } } @@ -1493,10 +1482,9 @@ UniValue listtransactions(const JSONRPCRequest& request) static UniValue listsinceblock(const JSONRPCRequest& request) { - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - const CWallet* const pwallet = wallet.get(); + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + if (!EnsureWalletIsAvailable(pwallet.get(), request.fHelp)) { return NullUniValue; } @@ -1553,12 +1541,12 @@ static UniValue listsinceblock(const JSONRPCRequest& request) }, }.Check(request); + const CWallet& wallet = *pwallet; // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now - pwallet->BlockUntilSyncedToCurrentChain(); + wallet.BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK(wallet.cs_wallet); // The way the 'height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. Optional<int> height = MakeOptional(false, int()); // Height of the specified block or the common ancestor, if the block provided was in a deactivated chain. @@ -1569,9 +1557,9 @@ static UniValue listsinceblock(const JSONRPCRequest& request) uint256 blockId; if (!request.params[0].isNull() && !request.params[0].get_str().empty()) { blockId = ParseHashV(request.params[0], "blockhash"); - height.emplace(); - altheight.emplace(); - if (!pwallet->chain().findCommonAncestor(blockId, pwallet->GetLastBlockHash(), /* ancestor out */ FoundBlock().height(*height), /* blockId out */ FoundBlock().height(*altheight))) { + height = int{}; + altheight = int{}; + if (!wallet.chain().findCommonAncestor(blockId, wallet.GetLastBlockHash(), /* ancestor out */ FoundBlock().height(*height), /* blockId out */ FoundBlock().height(*altheight))) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } } @@ -1584,21 +1572,21 @@ static UniValue listsinceblock(const JSONRPCRequest& request) } } - if (ParseIncludeWatchonly(request.params[2], *pwallet)) { + if (ParseIncludeWatchonly(request.params[2], wallet)) { filter |= ISMINE_WATCH_ONLY; } bool include_removed = (request.params[3].isNull() || request.params[3].get_bool()); - int depth = height ? pwallet->GetLastBlockHeight() + 1 - *height : -1; + int depth = height ? wallet.GetLastBlockHeight() + 1 - *height : -1; UniValue transactions(UniValue::VARR); - for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - CWalletTx tx = pairWtx.second; + for (const std::pair<const uint256, CWalletTx>& pairWtx : wallet.mapWallet) { + const CWalletTx& tx = pairWtx.second; if (depth == -1 || abs(tx.GetDepthInMainChain()) < depth) { - ListTransactions(*locked_chain, pwallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); + ListTransactions(&wallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); } } @@ -1607,15 +1595,15 @@ static UniValue listsinceblock(const JSONRPCRequest& request) UniValue removed(UniValue::VARR); while (include_removed && altheight && *altheight > *height) { CBlock block; - if (!pwallet->chain().findBlock(blockId, FoundBlock().data(block)) || block.IsNull()) { + if (!wallet.chain().findBlock(blockId, FoundBlock().data(block)) || block.IsNull()) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); } for (const CTransactionRef& tx : block.vtx) { - auto it = pwallet->mapWallet.find(tx->GetHash()); - if (it != pwallet->mapWallet.end()) { + auto it = wallet.mapWallet.find(tx->GetHash()); + if (it != wallet.mapWallet.end()) { // We want all transactions regardless of confirmation count to appear here, // even negative confirmation ones, hence the big negative. - ListTransactions(*locked_chain, pwallet, it->second, -100000000, true, removed, filter, nullptr /* filter_label */); + ListTransactions(&wallet, it->second, -100000000, true, removed, filter, nullptr /* filter_label */); } } blockId = block.hashPrevBlock; @@ -1623,7 +1611,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) } uint256 lastblock; - CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(pwallet->GetLastBlockHash(), pwallet->GetLastBlockHeight() + 1 - target_confirms, FoundBlock().hash(lastblock))); + CHECK_NONFATAL(wallet.chain().findAncestorByHeight(wallet.GetLastBlockHash(), wallet.GetLastBlockHeight() + 1 - target_confirms, FoundBlock().hash(lastblock))); UniValue ret(UniValue::VOBJ); ret.pushKV("transactions", transactions); @@ -1700,7 +1688,6 @@ static UniValue gettransaction(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); uint256 hash(ParseHashV(request.params[0], "txid")); @@ -1729,10 +1716,10 @@ static UniValue gettransaction(const JSONRPCRequest& request) if (wtx.IsFromMe(filter)) entry.pushKV("fee", ValueFromAmount(nFee)); - WalletTxToJSON(pwallet->chain(), *locked_chain, wtx, entry); + WalletTxToJSON(pwallet->chain(), wtx, entry); UniValue details(UniValue::VARR); - ListTransactions(*locked_chain, pwallet, wtx, 0, false, details, filter, nullptr /* filter_label */); + ListTransactions(pwallet, wtx, 0, false, details, filter, nullptr /* filter_label */); entry.pushKV("details", details); std::string strHex = EncodeHexTx(*wtx.tx, pwallet->chain().rpcSerializationFlags()); @@ -1776,7 +1763,6 @@ static UniValue abandontransaction(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); uint256 hash(ParseHashV(request.params[0], "txid")); @@ -1817,7 +1803,6 @@ static UniValue backupwallet(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); std::string strDest = request.params[0].get_str(); @@ -1855,7 +1840,6 @@ static UniValue keypoolrefill(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); } - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); // 0 is interpreted by TopUpKeyPool() as the default keypool size given by -keypool @@ -1908,8 +1892,10 @@ static UniValue walletpassphrase(const JSONRPCRequest& request) }.Check(request); int64_t nSleepTime; + int64_t relock_time; + // Prevent concurrent calls to walletpassphrase with the same wallet. + LOCK(pwallet->m_unlock_mutex); { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (!pwallet->IsCrypted()) { @@ -1946,6 +1932,7 @@ static UniValue walletpassphrase(const JSONRPCRequest& request) pwallet->TopUpKeyPool(); pwallet->nRelockTime = GetTime() + nSleepTime; + relock_time = pwallet->nRelockTime; } // rpcRunLater must be called without cs_wallet held otherwise a deadlock @@ -1957,9 +1944,11 @@ static UniValue walletpassphrase(const JSONRPCRequest& request) // wallet before the following callback is called. If a valid shared pointer // is acquired in the callback then the wallet is still loaded. std::weak_ptr<CWallet> weak_wallet = wallet; - pwallet->chain().rpcRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), [weak_wallet] { + pwallet->chain().rpcRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), [weak_wallet, relock_time] { if (auto shared_wallet = weak_wallet.lock()) { LOCK(shared_wallet->cs_wallet); + // Skip if this is not the most recent rpcRunLater callback. + if (shared_wallet->nRelockTime != relock_time) return; shared_wallet->Lock(); shared_wallet->nRelockTime = 0; } @@ -1991,7 +1980,6 @@ static UniValue walletpassphrasechange(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (!pwallet->IsCrypted()) { @@ -2047,7 +2035,6 @@ static UniValue walletlock(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (!pwallet->IsCrypted()) { @@ -2094,7 +2081,6 @@ static UniValue encryptwallet(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { @@ -2173,7 +2159,6 @@ static UniValue lockunspent(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); RPCTypeCheckArgument(request.params[0], UniValue::VBOOL); @@ -2286,7 +2271,6 @@ static UniValue listlockunspent(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); std::vector<COutPoint> vOutpts; @@ -2329,7 +2313,6 @@ static UniValue settxfee(const JSONRPCRequest& request) }, }.Check(request); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); CAmount nAmount = AmountFromValue(request.params[0]); @@ -2388,7 +2371,6 @@ static UniValue getbalances(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - auto locked_chain = wallet.chain().lock(); LOCK(wallet.cs_wallet); const auto bal = wallet.GetBalance(); @@ -2465,7 +2447,6 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); UniValue obj(UniValue::VOBJ); @@ -2615,14 +2596,14 @@ static UniValue loadwallet(const JSONRPCRequest& request) } } - std::string error; - std::vector<std::string> warning; - std::shared_ptr<CWallet> const wallet = LoadWallet(*g_rpc_chain, location, error, warning); - if (!wallet) throw JSONRPCError(RPC_WALLET_ERROR, error); + bilingual_str error; + std::vector<bilingual_str> warnings; + std::shared_ptr<CWallet> const wallet = LoadWallet(*g_rpc_chain, location, error, warnings); + if (!wallet) throw JSONRPCError(RPC_WALLET_ERROR, error.original); UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); - obj.pushKV("warning", Join(warning, "\n")); + obj.pushKV("warning", Join(warnings, Untranslated("\n")).original); return obj; } @@ -2731,12 +2712,12 @@ static UniValue createwallet(const JSONRPCRequest& request) } SecureString passphrase; passphrase.reserve(100); - std::vector<std::string> warnings; + std::vector<bilingual_str> warnings; if (!request.params[3].isNull()) { passphrase = request.params[3].get_str().c_str(); if (passphrase.empty()) { // Empty string means unencrypted - warnings.emplace_back("Empty string given as passphrase, wallet will not be encrypted."); + warnings.emplace_back(Untranslated("Empty string given as passphrase, wallet will not be encrypted.")); } } @@ -2745,16 +2726,17 @@ static UniValue createwallet(const JSONRPCRequest& request) } if (!request.params[5].isNull() && request.params[5].get_bool()) { flags |= WALLET_FLAG_DESCRIPTORS; + warnings.emplace_back(Untranslated("Wallet is an experimental descriptor wallet")); } - std::string error; + bilingual_str error; std::shared_ptr<CWallet> wallet; WalletCreationStatus status = CreateWallet(*g_rpc_chain, passphrase, flags, request.params[0].get_str(), error, warnings, wallet); switch (status) { case WalletCreationStatus::CREATION_FAILED: - throw JSONRPCError(RPC_WALLET_ERROR, error); + throw JSONRPCError(RPC_WALLET_ERROR, error.original); case WalletCreationStatus::ENCRYPTION_FAILED: - throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, error); + throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, error.original); case WalletCreationStatus::SUCCESS: break; // no default case, so the compiler can warn about missing cases @@ -2762,7 +2744,7 @@ static UniValue createwallet(const JSONRPCRequest& request) UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); - obj.pushKV("warning", Join(warnings, "\n")); + obj.pushKV("warning", Join(warnings, Untranslated("\n")).original); return obj; } @@ -2940,9 +2922,8 @@ static UniValue listunspent(const JSONRPCRequest& request) cctl.m_avoid_address_reuse = false; cctl.m_min_depth = nMinDepth; cctl.m_max_depth = nMaxDepth; - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->AvailableCoins(*locked_chain, vecOutputs, !include_unsafe, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount); + pwallet->AvailableCoins(vecOutputs, !include_unsafe, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount); } LOCK(pwallet->cs_wallet); @@ -3135,10 +3116,10 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f setSubtractFeeFromOutputs.insert(pos); } - std::string strFailReason; + bilingual_str error; - if (!pwallet->FundTransaction(tx, fee_out, change_position, strFailReason, lockUnspents, setSubtractFeeFromOutputs, coinControl)) { - throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); + if (!pwallet->FundTransaction(tx, fee_out, change_position, error, lockUnspents, setSubtractFeeFromOutputs, coinControl)) { + throw JSONRPCError(RPC_WALLET_ERROR, error.original); } } @@ -3312,7 +3293,6 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) } // Sign the transaction - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -3441,12 +3421,11 @@ static UniValue bumpfee(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); - std::vector<std::string> errors; + std::vector<bilingual_str> errors; CAmount old_fee; CAmount new_fee; CMutableTransaction mtx; @@ -3456,19 +3435,19 @@ static UniValue bumpfee(const JSONRPCRequest& request) if (res != feebumper::Result::OK) { switch(res) { case feebumper::Result::INVALID_ADDRESS_OR_KEY: - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errors[0]); + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errors[0].original); break; case feebumper::Result::INVALID_REQUEST: - throw JSONRPCError(RPC_INVALID_REQUEST, errors[0]); + throw JSONRPCError(RPC_INVALID_REQUEST, errors[0].original); break; case feebumper::Result::INVALID_PARAMETER: - throw JSONRPCError(RPC_INVALID_PARAMETER, errors[0]); + throw JSONRPCError(RPC_INVALID_PARAMETER, errors[0].original); break; case feebumper::Result::WALLET_ERROR: - throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); + throw JSONRPCError(RPC_WALLET_ERROR, errors[0].original); break; default: - throw JSONRPCError(RPC_MISC_ERROR, errors[0]); + throw JSONRPCError(RPC_MISC_ERROR, errors[0].original); break; } } @@ -3484,7 +3463,7 @@ static UniValue bumpfee(const JSONRPCRequest& request) uint256 txid; if (feebumper::CommitTransaction(*pwallet, hash, std::move(mtx), errors, txid) != feebumper::Result::OK) { - throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); + throw JSONRPCError(RPC_WALLET_ERROR, errors[0].original); } result.pushKV("txid", txid.GetHex()); @@ -3502,8 +3481,8 @@ static UniValue bumpfee(const JSONRPCRequest& request) result.pushKV("origfee", ValueFromAmount(old_fee)); result.pushKV("fee", ValueFromAmount(new_fee)); UniValue result_errors(UniValue::VARR); - for (const std::string& error : errors) { - result_errors.push_back(error); + for (const bilingual_str& error : errors) { + result_errors.push_back(error.original); } result.pushKV("errors", result_errors); @@ -3548,7 +3527,6 @@ UniValue rescanblockchain(const JSONRPCRequest& request) Optional<int> stop_height; uint256 start_block; { - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); int tip_height = pwallet->GetLastBlockHeight(); @@ -3563,8 +3541,7 @@ UniValue rescanblockchain(const JSONRPCRequest& request) stop_height = request.params[1].get_int(); if (*stop_height < 0 || *stop_height > tip_height) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stop_height"); - } - else if (*stop_height < start_height) { + } else if (*stop_height < start_height) { throw JSONRPCError(RPC_INVALID_PARAMETER, "stop_height must be greater than start_height"); } } @@ -4003,15 +3980,10 @@ UniValue sethdseed(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet, true); - if (pwallet->chain().isInitialBlockDownload()) { - throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Cannot set a new HD seed while still in Initial Block Download"); - } - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed to a wallet with private keys disabled"); } - auto locked_chain = pwallet->chain().lock(); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); // Do not do anything to non-HD wallets @@ -4271,12 +4243,12 @@ static UniValue upgradewallet(const JSONRPCRequest& request) version = request.params[0].get_int(); } - std::string error; - std::vector<std::string> warnings; + bilingual_str error; + std::vector<bilingual_str> warnings; if (!pwallet->UpgradeWallet(version, error, warnings)) { - throw JSONRPCError(RPC_WALLET_ERROR, error); + throw JSONRPCError(RPC_WALLET_ERROR, error.original); } - return error; + return error.original; } UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp new file mode 100644 index 0000000000..70067ebef0 --- /dev/null +++ b/src/wallet/salvage.cpp @@ -0,0 +1,150 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <fs.h> +#include <streams.h> +#include <wallet/salvage.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> + +/* End of headers, beginning of key/value data */ +static const char *HEADER_END = "HEADER=END"; +/* End of key/value data */ +static const char *DATA_END = "DATA=END"; +typedef std::pair<std::vector<unsigned char>, std::vector<unsigned char> > KeyValPair; + +bool RecoverDatabaseFile(const fs::path& file_path) +{ + std::string filename; + std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); + + // Recovery procedure: + // move wallet file to walletfilename.timestamp.bak + // Call Salvage with fAggressive=true to + // get as much data as possible. + // Rewrite salvaged data to fresh wallet file + // Set -rescan so any missing transactions will be + // found. + int64_t now = GetTime(); + std::string newFilename = strprintf("%s.%d.bak", filename, now); + + int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr, + newFilename.c_str(), DB_AUTO_COMMIT); + if (result == 0) + LogPrintf("Renamed %s to %s\n", filename, newFilename); + else + { + LogPrintf("Failed to rename %s to %s\n", filename, newFilename); + return false; + } + + /** + * Salvage data from a file. The DB_AGGRESSIVE flag is being used (see berkeley DB->verify() method documentation). + * key/value pairs are appended to salvagedData which are then written out to a new wallet file. + * NOTE: reads the entire database into memory, so cannot be used + * for huge databases. + */ + std::vector<KeyValPair> salvagedData; + + std::stringstream strDump; + + Db db(env->dbenv.get(), 0); + result = db.verify(newFilename.c_str(), nullptr, &strDump, DB_SALVAGE | DB_AGGRESSIVE); + if (result == DB_VERIFY_BAD) { + LogPrintf("Salvage: Database salvage found errors, all data may not be recoverable.\n"); + } + if (result != 0 && result != DB_VERIFY_BAD) { + LogPrintf("Salvage: Database salvage failed with result %d.\n", result); + return false; + } + + // Format of bdb dump is ascii lines: + // header lines... + // HEADER=END + // hexadecimal key + // hexadecimal value + // ... repeated + // DATA=END + + std::string strLine; + while (!strDump.eof() && strLine != HEADER_END) + getline(strDump, strLine); // Skip past header + + std::string keyHex, valueHex; + while (!strDump.eof() && keyHex != DATA_END) { + getline(strDump, keyHex); + if (keyHex != DATA_END) { + if (strDump.eof()) + break; + getline(strDump, valueHex); + if (valueHex == DATA_END) { + LogPrintf("Salvage: WARNING: Number of keys in data does not match number of values.\n"); + break; + } + salvagedData.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex))); + } + } + + bool fSuccess; + if (keyHex != DATA_END) { + LogPrintf("Salvage: WARNING: Unexpected end of file while reading salvage output.\n"); + fSuccess = false; + } else { + fSuccess = (result == 0); + } + + if (salvagedData.empty()) + { + LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); + return false; + } + LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); + + std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); + int ret = pdbCopy->open(nullptr, // Txn pointer + filename.c_str(), // Filename + "main", // Logical db name + DB_BTREE, // Database type + DB_CREATE, // Flags + 0); + if (ret > 0) { + LogPrintf("Cannot create database file %s\n", filename); + pdbCopy->close(0); + return false; + } + + DbTxn* ptxn = env->TxnBegin(); + CWallet dummyWallet(nullptr, WalletLocation(), WalletDatabase::CreateDummy()); + for (KeyValPair& row : salvagedData) + { + /* Filter for only private key type KV pairs to be added to the salvaged wallet */ + CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); + CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); + std::string strType, strErr; + bool fReadOK; + { + // Required in LoadKeyMetadata(): + LOCK(dummyWallet.cs_wallet); + fReadOK = ReadKeyValue(&dummyWallet, ssKey, ssValue, strType, strErr); + } + if (!WalletBatch::IsKeyType(strType) && strType != DBKeys::HDCHAIN) { + continue; + } + if (!fReadOK) + { + LogPrintf("WARNING: WalletBatch::Recover skipping %s: %s\n", strType, strErr); + continue; + } + Dbt datKey(&row.first[0], row.first.size()); + Dbt datValue(&row.second[0], row.second.size()); + int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); + if (ret2 > 0) + fSuccess = false; + } + ptxn->commit(0); + pdbCopy->close(0); + + return fSuccess; +} diff --git a/src/wallet/salvage.h b/src/wallet/salvage.h new file mode 100644 index 0000000000..e361930f5e --- /dev/null +++ b/src/wallet/salvage.h @@ -0,0 +1,14 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_SALVAGE_H +#define BITCOIN_WALLET_SALVAGE_H + +#include <fs.h> +#include <streams.h> + +bool RecoverDatabaseFile(const fs::path& file_path); + +#endif // BITCOIN_WALLET_SALVAGE_H diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index ecb95d599d..8a2a798644 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -12,6 +12,9 @@ #include <util/translation.h> #include <wallet/scriptpubkeyman.h> +//! Value for the first BIP 32 hardened derivation. Can be used as a bit mask and as a value. See BIP 32 for more details. +const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; + bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { LOCK(cs_KeyStore); @@ -220,6 +223,7 @@ bool LegacyScriptPubKeyMan::CheckDecryptionKey(const CKeyingMaterial& master_key bool keyPass = mapCryptedKeys.empty(); // Always pass when there are no encrypted keys bool keyFail = false; CryptedKeyMap::const_iterator mi = mapCryptedKeys.begin(); + WalletBatch batch(m_storage.GetDatabase()); for (; mi != mapCryptedKeys.end(); ++mi) { const CPubKey &vchPubKey = (*mi).second.first; @@ -233,6 +237,10 @@ bool LegacyScriptPubKeyMan::CheckDecryptionKey(const CKeyingMaterial& master_key keyPass = true; if (fDecryptionThoroughlyChecked) break; + else { + // Rewrite these encrypted keys with checksums + batch.WriteCryptedKey(vchPubKey, vchCryptedSecret, mapKeyMetadata[vchPubKey.GetID()]); + } } if (keyPass && keyFail) { @@ -290,6 +298,43 @@ bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool i return true; } +bool LegacyScriptPubKeyMan::TopUpInactiveHDChain(const CKeyID seed_id, int64_t index, bool internal) +{ + LOCK(cs_KeyStore); + + if (m_storage.IsLocked()) return false; + + auto it = m_inactive_hd_chains.find(seed_id); + if (it == m_inactive_hd_chains.end()) { + return false; + } + + CHDChain& chain = it->second; + + // Top up key pool + int64_t target_size = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); + + // "size" of the keypools. Not really the size, actually the difference between index and the chain counter + // Since chain counter is 1 based and index is 0 based, one of them needs to be offset by 1. + int64_t kp_size = (internal ? chain.nInternalChainCounter : chain.nExternalChainCounter) - (index + 1); + + // make sure the keypool fits the user-selected target (-keypool) + int64_t missing = std::max(target_size - kp_size, (int64_t) 0); + + if (missing > 0) { + WalletBatch batch(m_storage.GetDatabase()); + for (int64_t i = missing; i > 0; --i) { + GenerateNewKey(batch, chain, internal); + } + if (internal) { + WalletLogPrintf("inactive seed with id %s added %d internal keys\n", HexStr(seed_id), missing); + } else { + WalletLogPrintf("inactive seed with id %s added %d keys\n", HexStr(seed_id), missing); + } + } + return true; +} + void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) { LOCK(cs_KeyStore); @@ -297,13 +342,28 @@ void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) for (const auto& keyid : GetAffectedKeys(script, *this)) { std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); if (mi != m_pool_key_to_index.end()) { - WalletLogPrintf("%s: Detected a used keypool key, mark all keypool key up to this key as used\n", __func__); + WalletLogPrintf("%s: Detected a used keypool key, mark all keypool keys up to this key as used\n", __func__); MarkReserveKeysAsUsed(mi->second); if (!TopUp()) { WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); } } + + // Find the key's metadata and check if it's seed id (if it has one) is inactive, i.e. it is not the current m_hd_chain seed id. + // If so, TopUp the inactive hd chain + auto it = mapKeyMetadata.find(keyid); + if (it != mapKeyMetadata.end()){ + CKeyMetadata meta = it->second; + if (!meta.hd_seed_id.IsNull() && meta.hd_seed_id != m_hd_chain.seed_id) { + bool internal = (meta.key_origin.path[1] & ~BIP32_HARDENED_KEY_LIMIT) != 0; + int64_t index = meta.key_origin.path[2] & ~BIP32_HARDENED_KEY_LIMIT; + + if (!TopUpInactiveHDChain(meta.hd_seed_id, index, internal)) { + WalletLogPrintf("%s: Adding inactive seed keys failed\n", __func__); + } + } + } } } @@ -357,7 +417,7 @@ bool LegacyScriptPubKeyMan::SetupGeneration(bool force) bool LegacyScriptPubKeyMan::IsHDEnabled() const { - return !hdChain.seed_id.IsNull(); + return !m_hd_chain.seed_id.IsNull(); } bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) const @@ -377,10 +437,9 @@ bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) const return keypool_has_keys; } -bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) +bool LegacyScriptPubKeyMan::Upgrade(int prev_version, bilingual_str& error) { LOCK(cs_KeyStore); - error = ""; bool hd_upgrade = false; bool split_upgrade = false; if (m_storage.CanSupportFeature(FEATURE_HD) && !IsHDEnabled()) { @@ -405,7 +464,7 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) // Regenerate the keypool if upgraded to HD if (hd_upgrade) { if (!TopUp()) { - error = _("Unable to generate keys").translated; + error = _("Unable to generate keys"); return false; } } @@ -714,8 +773,13 @@ bool LegacyScriptPubKeyMan::AddKeyPubKeyInner(const CKey& key, const CPubKey &pu return true; } -bool LegacyScriptPubKeyMan::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) +bool LegacyScriptPubKeyMan::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret, bool checksum_valid) { + // Set fDecryptionThoroughlyChecked to false when the checksum is invalid + if (!checksum_valid) { + fDecryptionThoroughlyChecked = false; + } + return AddCryptedKeyInner(vchPubKey, vchCryptedSecret); } @@ -839,10 +903,27 @@ bool LegacyScriptPubKeyMan::AddWatchOnly(const CScript& dest, int64_t nCreateTim void LegacyScriptPubKeyMan::SetHDChain(const CHDChain& chain, bool memonly) { LOCK(cs_KeyStore); - if (!memonly && !WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) - throw std::runtime_error(std::string(__func__) + ": writing chain failed"); + // memonly == true means we are loading the wallet file + // memonly == false means that the chain is actually being changed + if (!memonly) { + // Store the new chain + if (!WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) { + throw std::runtime_error(std::string(__func__) + ": writing chain failed"); + } + // When there's an old chain, add it as an inactive chain as we are now rotating hd chains + if (!m_hd_chain.seed_id.IsNull()) { + AddInactiveHDChain(m_hd_chain); + } + } + + m_hd_chain = chain; +} - hdChain = chain; +void LegacyScriptPubKeyMan::AddInactiveHDChain(const CHDChain& chain) +{ + LOCK(cs_KeyStore); + assert(!chain.seed_id.IsNull()); + m_inactive_hd_chains[chain.seed_id] = chain; } bool LegacyScriptPubKeyMan::HaveKey(const CKeyID &address) const @@ -921,7 +1002,7 @@ bool LegacyScriptPubKeyMan::GetPubKey(const CKeyID &address, CPubKey& vchPubKeyO return GetWatchPubKey(address, vchPubKeyOut); } -CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, bool internal) +CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, CHDChain& hd_chain, bool internal) { assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); @@ -936,7 +1017,7 @@ CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, bool internal) // use HD key derivation if HD was enabled during wallet creation and a seed is present if (IsHDEnabled()) { - DeriveNewChildKey(batch, metadata, secret, (m_storage.CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); + DeriveNewChildKey(batch, metadata, secret, hd_chain, (m_storage.CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); } else { secret.MakeNewKey(fCompressed); } @@ -958,9 +1039,7 @@ CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, bool internal) return pubkey; } -const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; - -void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, bool internal) +void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, CHDChain& hd_chain, bool internal) { // for now we use a fixed keypath scheme of m/0'/0'/k CKey seed; //seed (256bit) @@ -970,7 +1049,7 @@ void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& CExtKey childKey; //key at m/0'/0'/<n>' // try to get the seed - if (!GetKey(hdChain.seed_id, seed)) + if (!GetKey(hd_chain.seed_id, seed)) throw std::runtime_error(std::string(__func__) + ": seed not found"); masterKey.SetSeed(seed.begin(), seed.size()); @@ -989,30 +1068,30 @@ void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 if (internal) { - chainChildKey.Derive(childKey, hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/1'/" + ToString(hdChain.nInternalChainCounter) + "'"; + chainChildKey.Derive(childKey, hd_chain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/1'/" + ToString(hd_chain.nInternalChainCounter) + "'"; metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); metadata.key_origin.path.push_back(1 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - hdChain.nInternalChainCounter++; + metadata.key_origin.path.push_back(hd_chain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + hd_chain.nInternalChainCounter++; } else { - chainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/0'/" + ToString(hdChain.nExternalChainCounter) + "'"; + chainChildKey.Derive(childKey, hd_chain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/0'/" + ToString(hd_chain.nExternalChainCounter) + "'"; metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - hdChain.nExternalChainCounter++; + metadata.key_origin.path.push_back(hd_chain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + hd_chain.nExternalChainCounter++; } } while (HaveKey(childKey.key.GetPubKey().GetID())); secret = childKey.key; - metadata.hd_seed_id = hdChain.seed_id; + metadata.hd_seed_id = hd_chain.seed_id; CKeyID master_id = masterKey.key.GetPubKey().GetID(); std::copy(master_id.begin(), master_id.begin() + 4, metadata.key_origin.fingerprint); metadata.has_key_origin = true; // update the chain model in the database - if (!batch.WriteHDChain(hdChain)) - throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); + if (hd_chain.seed_id == m_hd_chain.seed_id && !batch.WriteHDChain(hd_chain)) + throw std::runtime_error(std::string(__func__) + ": writing HD chain model failed"); } void LegacyScriptPubKeyMan::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) @@ -1167,7 +1246,7 @@ bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) internal = true; } - CPubKey pubkey(GenerateNewKey(batch, internal)); + CPubKey pubkey(GenerateNewKey(batch, m_hd_chain, internal)); AddKeypoolPubkeyWithDB(pubkey, internal, batch); } if (missingInternal + missingExternal > 0) { @@ -1240,7 +1319,7 @@ bool LegacyScriptPubKeyMan::GetKeyFromPool(CPubKey& result, const OutputType typ if (!ReserveKeyFromKeyPool(nIndex, keypool, internal) && !m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { if (m_storage.IsLocked()) return false; WalletBatch batch(m_storage.GetDatabase()); - result = GenerateNewKey(batch, internal); + result = GenerateNewKey(batch, m_hd_chain, internal); return true; } KeepDestination(nIndex, type); @@ -1498,7 +1577,7 @@ std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const return set_address; } -void LegacyScriptPubKeyMan::SetType(OutputType type, bool internal) {} +void LegacyScriptPubKeyMan::SetInternal(bool internal) {} bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { @@ -1510,7 +1589,9 @@ bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDest { LOCK(cs_desc_man); assert(m_wallet_descriptor.descriptor->IsSingleType()); // This is a combo descriptor which should not be an active descriptor - if (type != m_address_type) { + Optional<OutputType> desc_addr_type = m_wallet_descriptor.descriptor->GetOutputType(); + assert(desc_addr_type); + if (type != *desc_addr_type) { throw std::runtime_error(std::string(__func__) + ": Types are inconsistent"); } @@ -1778,7 +1859,7 @@ bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const } } -bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key) +bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type) { LOCK(cs_desc_man); assert(m_storage.IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); @@ -1795,7 +1876,7 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ // Build descriptor string std::string desc_prefix; std::string desc_suffix = "/*)"; - switch (m_address_type) { + switch (addr_type) { case OutputType::LEGACY: { desc_prefix = "pkh(" + xpub + "/44'"; break; @@ -2077,9 +2158,8 @@ uint256 DescriptorScriptPubKeyMan::GetID() const return id; } -void DescriptorScriptPubKeyMan::SetType(OutputType type, bool internal) +void DescriptorScriptPubKeyMan::SetInternal(bool internal) { - this->m_address_type = type; this->m_internal = internal; } diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 3117b13d35..d62d30f339 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -18,7 +18,10 @@ #include <boost/signals2/signal.hpp> +#include <unordered_map> + enum class OutputType; +struct bilingual_str; // Wallet storage things that ScriptPubKeyMans need in order to be able to store things to the wallet database. // It provides access to things that are part of the entire wallet and not specific to a ScriptPubKeyMan such as @@ -109,40 +112,52 @@ public: CKeyPool(); CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn); - ADD_SERIALIZE_METHODS; + template<typename Stream> + void Serialize(Stream& s) const + { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) { + s << nVersion; + } + s << nTime << vchPubKey << fInternal << m_pre_split; + } - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { + template<typename Stream> + void Unserialize(Stream& s) + { int nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(nVersion); - READWRITE(nTime); - READWRITE(vchPubKey); - if (ser_action.ForRead()) { - try { - READWRITE(fInternal); - } - catch (std::ios_base::failure&) { - /* flag as external address if we can't read the internal boolean - (this will be the case for any wallet before the HD chain split version) */ - fInternal = false; - } - try { - READWRITE(m_pre_split); - } - catch (std::ios_base::failure&) { - /* flag as postsplit address if we can't read the m_pre_split boolean - (this will be the case for any wallet that upgrades to HD chain split)*/ - m_pre_split = false; - } + if (!(s.GetType() & SER_GETHASH)) { + s >> nVersion; + } + s >> nTime >> vchPubKey; + try { + s >> fInternal; + } catch (std::ios_base::failure&) { + /* flag as external address if we can't read the internal boolean + (this will be the case for any wallet before the HD chain split version) */ + fInternal = false; } - else { - READWRITE(fInternal); - READWRITE(m_pre_split); + try { + s >> m_pre_split; + } catch (std::ios_base::failure&) { + /* flag as postsplit address if we can't read the m_pre_split boolean + (this will be the case for any wallet that upgrades to HD chain split) */ + m_pre_split = false; } } }; +class KeyIDHasher +{ +public: + KeyIDHasher() {} + + size_t operator()(const CKeyID& id) const + { + return id.GetUint64(0); + } +}; + /* * A class implementing ScriptPubKeyMan manages some (or all) scriptPubKeys used in a wallet. * It contains the scripts and keys related to the scriptPubKeys it manages. @@ -191,7 +206,7 @@ public: virtual bool CanGetAddresses(bool internal = false) const { return false; } /** Upgrades the wallet to the specified version */ - virtual bool Upgrade(int prev_version, std::string& error) { return false; } + virtual bool Upgrade(int prev_version, bilingual_str& error) { return false; } virtual bool HavePrivateKeys() const { return false; } @@ -223,7 +238,7 @@ public: virtual uint256 GetID() const { return uint256(); } - virtual void SetType(OutputType type, bool internal) {} + virtual void SetInternal(bool internal) {} /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ template<typename... Params> @@ -242,7 +257,7 @@ class LegacyScriptPubKeyMan : public ScriptPubKeyMan, public FillableSigningProv { private: //! keeps track of whether Unlock has run a thorough check before - bool fDecryptionThoroughlyChecked = false; + bool fDecryptionThoroughlyChecked = true; using WatchOnlySet = std::set<CScript>; using WatchKeyMap = std::map<CKeyID, CPubKey>; @@ -287,10 +302,11 @@ private: bool AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info); /* the HD chain data model (external chain counters) */ - CHDChain hdChain; + CHDChain m_hd_chain; + std::unordered_map<CKeyID, CHDChain, KeyIDHasher> m_inactive_hd_chains; /* HD derive new child key (on internal or external chain) */ - void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, CHDChain& hd_chain, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_KeyStore); std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_KeyStore); @@ -319,6 +335,18 @@ private: */ bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); + /** + * Like TopUp() but adds keys for inactive HD chains. + * Ensures that there are at least -keypool number of keys derived after the given index. + * + * @param seed_id the CKeyID for the HD seed. + * @param index the index to start generating keys from + * @param internal whether the internal chain should be used. true for internal chain, false for external chain. + * + * @return true if seed was found and keys were derived. false if unable to derive seeds + */ + bool TopUpInactiveHDChain(const CKeyID seed_id, int64_t index, bool internal); + public: using ScriptPubKeyMan::ScriptPubKeyMan; @@ -343,7 +371,7 @@ public: bool SetupGeneration(bool force = false) override; - bool Upgrade(int prev_version, std::string& error) override; + bool Upgrade(int prev_version, bilingual_str& error) override; bool HavePrivateKeys() const override; @@ -369,7 +397,7 @@ public: uint256 GetID() const override; - void SetType(OutputType type, bool internal) override; + void SetInternal(bool internal) override; // Map from Key ID to key metadata. std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_KeyStore); @@ -384,7 +412,7 @@ public: //! Adds an encrypted key to the store, and saves it to disk. bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); //! Adds an encrypted key to the store, without saving it to disk (used by LoadWallet) - bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); + bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret, bool checksum_valid); void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); //! Adds a CScript to the store bool LoadCScript(const CScript& redeemScript); @@ -392,11 +420,12 @@ public: void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata); void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata); //! Generate a new key - CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + CPubKey GenerateNewKey(WalletBatch& batch, CHDChain& hd_chain, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); /* Set the HD chain model (chain child index counters) */ void SetHDChain(const CHDChain& chain, bool memonly); - const CHDChain& GetHDChain() const { return hdChain; } + const CHDChain& GetHDChain() const { return m_hd_chain; } + void AddInactiveHDChain(const CHDChain& chain); //! Adds a watch-only address to the store, without saving it to disk (used by LoadWallet) bool LoadWatchOnly(const CScript &dest); @@ -496,14 +525,11 @@ private: PubKeyMap m_map_pubkeys GUARDED_BY(cs_desc_man); int32_t m_max_cached_index = -1; - OutputType m_address_type; - bool m_internal; + bool m_internal = false; KeyMap m_map_keys GUARDED_BY(cs_desc_man); CryptedKeyMap m_map_crypted_keys GUARDED_BY(cs_desc_man); - bool SetCrypted(); - //! keeps track of whether Unlock has run a thorough check before bool m_decryption_thoroughly_checked = false; @@ -523,9 +549,9 @@ public: : ScriptPubKeyMan(storage), m_wallet_descriptor(descriptor) {} - DescriptorScriptPubKeyMan(WalletStorage& storage, OutputType address_type, bool internal) + DescriptorScriptPubKeyMan(WalletStorage& storage, bool internal) : ScriptPubKeyMan(storage), - m_address_type(address_type), m_internal(internal) + m_internal(internal) {} mutable RecursiveMutex cs_desc_man; @@ -550,7 +576,7 @@ public: bool IsHDEnabled() const override; //! Setup descriptors based on the given CExtkey - bool SetupDescriptorGeneration(const CExtKey& master_key); + bool SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type); bool HavePrivateKeys() const override; @@ -574,7 +600,7 @@ public: uint256 GetID() const override; - void SetType(OutputType type, bool internal) override; + void SetInternal(bool internal) override; void SetCache(const DescriptorCache& cache); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 66f4542cf9..657d0828f2 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -24,8 +24,6 @@ BOOST_FIXTURE_TEST_SUITE(coinselector_tests, WalletTestingSetup) // we repeat those tests this many times and only complain if all iterations of the test fail #define RANDOM_REPEATS 5 -std::vector<std::unique_ptr<CWalletTx>> wtxn; - typedef std::set<CInputCoin> CoinSet; static std::vector<COutput> vCoins; @@ -74,16 +72,14 @@ static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bo // so stop vin being empty, and cache a non-zero Debit to fake out IsFromMe() tx.vin.resize(1); } - std::unique_ptr<CWalletTx> wtx = MakeUnique<CWalletTx>(&wallet, MakeTransactionRef(std::move(tx))); + CWalletTx* wtx = wallet.AddToWallet(MakeTransactionRef(std::move(tx)), /* confirm= */ {}); if (fIsFromMe) { wtx->m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); wtx->m_is_cache_empty = false; } - COutput output(wtx.get(), nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); + COutput output(wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); vCoins.push_back(output); - wallet.AddToWallet(*wtx.get()); - wtxn.emplace_back(std::move(wtx)); } static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) { @@ -93,7 +89,6 @@ static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = fa static void empty_wallet(void) { vCoins.clear(); - wtxn.clear(); balance = 0; } diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 35860577cd..b4c65a8665 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -22,14 +22,12 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); CTransactionRef prev_tx1; s_prev_tx1 >> prev_tx1; - CWalletTx prev_wtx1(&m_wallet, prev_tx1); - m_wallet.mapWallet.emplace(prev_wtx1.GetHash(), std::move(prev_wtx1)); + m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx1->GetHash()), std::forward_as_tuple(&m_wallet, prev_tx1)); CDataStream s_prev_tx2(ParseHex("0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f618765000000"), SER_NETWORK, PROTOCOL_VERSION); CTransactionRef prev_tx2; s_prev_tx2 >> prev_tx2; - CWalletTx prev_wtx2(&m_wallet, prev_tx2); - m_wallet.mapWallet.emplace(prev_wtx2.GetHash(), std::move(prev_wtx2)); + m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(&m_wallet, prev_tx2)); // Add scripts CScript rs1; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 2bdeb89f3c..3654420eb2 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -4,6 +4,7 @@ #include <wallet/wallet.h> +#include <future> #include <memory> #include <stdint.h> #include <vector> @@ -12,7 +13,10 @@ #include <node/context.h> #include <policy/policy.h> #include <rpc/server.h> +#include <test/util/logging.h> #include <test/util/setup_common.h> +#include <util/ref.h> +#include <util/translation.h> #include <validation.h> #include <wallet/coincontrol.h> #include <wallet/test/wallet_test_fixture.h> @@ -26,6 +30,36 @@ extern UniValue importwallet(const JSONRPCRequest& request); BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) +static std::shared_ptr<CWallet> TestLoadWallet(interfaces::Chain& chain) +{ + bilingual_str error; + std::vector<bilingual_str> warnings; + auto wallet = CWallet::CreateWalletFromFile(chain, WalletLocation(""), error, warnings); + wallet->postInitProcess(); + return wallet; +} + +static void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet) +{ + SyncWithValidationInterfaceQueue(); + wallet->m_chain_notifications_handler.reset(); + UnloadWallet(std::move(wallet)); +} + +static CMutableTransaction TestSimpleSpend(const CTransaction& from, uint32_t index, const CKey& key, const CScript& pubkey) +{ + CMutableTransaction mtx; + mtx.vout.push_back({from.vout[index].nValue - DEFAULT_TRANSACTION_MAXFEE, pubkey}); + mtx.vin.push_back({CTxIn{from.GetHash(), index}}); + FillableSigningProvider keystore; + keystore.AddKey(key); + std::map<COutPoint, Coin> coins; + coins[mtx.vin[0].prevout].out = from.vout[index]; + std::map<int, std::string> input_errors; + BOOST_CHECK(SignTransaction(mtx, &keystore, coins, SIGHASH_ALL, input_errors)); + return mtx; +} + static void AddKey(CWallet& wallet, const CKey& key) { auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); @@ -43,8 +77,6 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) NodeContext node; auto chain = interfaces::MakeChain(node); - auto locked_chain = chain->lock(); - LockAssertion lock(::cs_main); // Verify ScanForWalletTransactions fails to read an unknown start block. { @@ -84,7 +116,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) } // Prune the older block file. - PruneOneBlockFile(oldTip->GetBlockPos().nFile); + { + LOCK(cs_main); + EnsureChainman(m_node).PruneOneBlockFile(oldTip->GetBlockPos().nFile); + } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); // Verify ScanForWalletTransactions only picks transactions in the new block @@ -107,7 +142,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) } // Prune the remaining block file. - PruneOneBlockFile(newTip->GetBlockPos().nFile); + { + LOCK(cs_main); + EnsureChainman(m_node).PruneOneBlockFile(newTip->GetBlockPos().nFile); + } UnlinkPrunedFiles({newTip->GetBlockPos().nFile}); // Verify ScanForWalletTransactions scans no blocks. @@ -139,11 +177,12 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) NodeContext node; auto chain = interfaces::MakeChain(node); - auto locked_chain = chain->lock(); - LockAssertion lock(::cs_main); // Prune the older block file. - PruneOneBlockFile(oldTip->GetBlockPos().nFile); + { + LOCK(cs_main); + EnsureChainman(m_node).PruneOneBlockFile(oldTip->GetBlockPos().nFile); + } UnlinkPrunedFiles({oldTip->GetBlockPos().nFile}); // Verify importmulti RPC returns failure for a key whose creation time is @@ -170,7 +209,8 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1); key.pushKV("internal", UniValue(true)); keys.push_back(key); - JSONRPCRequest request; + util::Ref context; + JSONRPCRequest request(context); request.params.setArray(); request.params.push_back(keys); @@ -209,8 +249,6 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) NodeContext node; auto chain = interfaces::MakeChain(node); - auto locked_chain = chain->lock(); - LockAssertion lock(::cs_main); std::string backup_file = (GetDataDir() / "wallet.backup").string(); @@ -226,7 +264,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) AddWallet(wallet); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } - JSONRPCRequest request; + util::Ref context; + JSONRPCRequest request(context); request.params.setArray(); request.params.push_back(backup_file); @@ -241,7 +280,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); - JSONRPCRequest request; + util::Ref context; + JSONRPCRequest request(context); request.params.setArray(); request.params.push_back(backup_file); AddWallet(wallet); @@ -276,8 +316,6 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); CWalletTx wtx(&wallet, m_coinbase_txns.back()); - auto locked_chain = chain->lock(); - LockAssertion lock(::cs_main); LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -298,36 +336,26 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) { CMutableTransaction tx; + CWalletTx::Confirmation confirm; tx.nLockTime = lockTime; SetMockTime(mockTime); CBlockIndex* block = nullptr; if (blockTime > 0) { - auto locked_chain = wallet.chain().lock(); - LockAssertion lock(::cs_main); auto inserted = ::BlockIndex().emplace(GetRandHash(), new CBlockIndex); assert(inserted.second); const uint256& hash = inserted.first->first; block = inserted.first->second; block->nTime = blockTime; block->phashBlock = &hash; + confirm = {CWalletTx::Status::CONFIRMED, block->nHeight, hash, 0}; } - CWalletTx wtx(&wallet, MakeTransactionRef(tx)); - LOCK(cs_main); - LOCK(wallet.cs_wallet); // If transaction is already in map, to avoid inconsistencies, unconfirmation // is needed before confirm again with different block. - std::map<uint256, CWalletTx>::iterator it = wallet.mapWallet.find(wtx.GetHash()); - if (it != wallet.mapWallet.end()) { + return wallet.AddToWallet(MakeTransactionRef(tx), confirm, [&](CWalletTx& wtx, bool /* new_tx */) { wtx.setUnconfirmed(); - wallet.AddToWallet(wtx); - } - if (block) { - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, block->nHeight, block->GetBlockHash(), 0); - wtx.m_confirm = confirm; - } - wallet.AddToWallet(wtx); - return wallet.mapWallet.at(wtx.GetHash()).nTimeSmart; + return true; + })->nTimeSmart; } // Simple test to verify assignment of CWalletTx::nSmartTime value. Could be @@ -460,7 +488,7 @@ public: CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); { - LOCK2(::cs_main, wallet->cs_wallet); + LOCK2(wallet->cs_wallet, ::cs_main); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } bool firstRun; @@ -485,11 +513,10 @@ public: CTransactionRef tx; CAmount fee; int changePos = -1; - std::string error; + bilingual_str error; CCoinControl dummy; { - auto locked_chain = m_chain->lock(); - BOOST_CHECK(wallet->CreateTransaction(*locked_chain, {recipient}, tx, fee, changePos, error, dummy)); + BOOST_CHECK(wallet->CreateTransaction({recipient}, tx, fee, changePos, error, dummy)); } wallet->CommitTransaction(tx, {}, {}); CMutableTransaction blocktx; @@ -499,7 +526,6 @@ public: } CreateAndProcessBlock({CMutableTransaction(blocktx)}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - LOCK(cs_main); LOCK(wallet->cs_wallet); wallet->SetLastBlockProcessed(wallet->GetLastBlockHeight() + 1, ::ChainActive().Tip()->GetBlockHash()); auto it = wallet->mapWallet.find(tx->GetHash()); @@ -522,9 +548,8 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) // address. std::map<CTxDestination, std::vector<COutput>> list; { - auto locked_chain = m_chain->lock(); LOCK(wallet->cs_wallet); - list = wallet->ListCoins(*locked_chain); + list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); @@ -539,9 +564,8 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) // pubkey. AddTx(CRecipient{GetScriptForRawPubKey({}), 1 * COIN, false /* subtract fee */}); { - auto locked_chain = m_chain->lock(); LOCK(wallet->cs_wallet); - list = wallet->ListCoins(*locked_chain); + list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); @@ -549,10 +573,9 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) // Lock both coins. Confirm number of available coins drops to 0. { - auto locked_chain = m_chain->lock(); LOCK(wallet->cs_wallet); std::vector<COutput> available; - wallet->AvailableCoins(*locked_chain, available); + wallet->AvailableCoins(available); BOOST_CHECK_EQUAL(available.size(), 2U); } for (const auto& group : list) { @@ -562,18 +585,16 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) } } { - auto locked_chain = m_chain->lock(); LOCK(wallet->cs_wallet); std::vector<COutput> available; - wallet->AvailableCoins(*locked_chain, available); + wallet->AvailableCoins(available); BOOST_CHECK_EQUAL(available.size(), 0U); } // Confirm ListCoins still returns same result as before, despite coins // being locked. { - auto locked_chain = m_chain->lock(); LOCK(wallet->cs_wallet); - list = wallet->ListCoins(*locked_chain); + list = wallet->ListCoins(); } BOOST_CHECK_EQUAL(list.size(), 1U); BOOST_CHECK_EQUAL(boost::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); @@ -658,4 +679,116 @@ BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) BOOST_CHECK_EXCEPTION(vr >> w_desc, std::ios_base::failure, malformed_descriptor); } +//! Test CreateWalletFromFile function and its behavior handling potential race +//! conditions if it's called the same time an incoming transaction shows up in +//! the mempool or a new block. +//! +//! It isn't possible to verify there aren't race condition in every case, so +//! this test just checks two specific cases and ensures that timing of +//! notifications in these cases doesn't prevent the wallet from detecting +//! transactions. +//! +//! In the first case, block and mempool transactions are created before the +//! wallet is loaded, but notifications about these transactions are delayed +//! until after it is loaded. The notifications are superfluous in this case, so +//! the test verifies the transactions are detected before they arrive. +//! +//! In the second case, block and mempool transactions are created after the +//! wallet rescan and notifications are immediately synced, to verify the wallet +//! must already have a handler in place for them, and there's no gap after +//! rescanning where new transactions in new blocks could be lost. +BOOST_FIXTURE_TEST_CASE(CreateWalletFromFile, TestChain100Setup) +{ + // Create new wallet with known key and unload it. + auto chain = interfaces::MakeChain(m_node); + auto wallet = TestLoadWallet(*chain); + CKey key; + key.MakeNewKey(true); + AddKey(*wallet, key); + TestUnloadWallet(std::move(wallet)); + + + // Add log hook to detect AddToWallet events from rescans, blockConnected, + // and transactionAddedToMempool notifications + int addtx_count = 0; + DebugLogHelper addtx_counter("[default wallet] AddToWallet", [&](const std::string* s) { + if (s) ++addtx_count; + return false; + }); + + + bool rescan_completed = false; + DebugLogHelper rescan_check("[default wallet] Rescan completed", [&](const std::string* s) { + if (s) rescan_completed = true; + return false; + }); + + + // Block the queue to prevent the wallet receiving blockConnected and + // transactionAddedToMempool notifications, and create block and mempool + // transactions paying to the wallet + std::promise<void> promise; + CallFunctionInValidationInterfaceQueue([&promise] { + promise.get_future().wait(); + }); + std::string error; + m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + auto block_tx = TestSimpleSpend(*m_coinbase_txns[0], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + m_coinbase_txns.push_back(CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + auto mempool_tx = TestSimpleSpend(*m_coinbase_txns[1], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + BOOST_CHECK(chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); + + + // Reload wallet and make sure new transactions are detected despite events + // being blocked + wallet = TestLoadWallet(*chain); + BOOST_CHECK(rescan_completed); + BOOST_CHECK_EQUAL(addtx_count, 2); + { + LOCK(wallet->cs_wallet); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(mempool_tx.GetHash()), 1U); + } + + + // Unblock notification queue and make sure stale blockConnected and + // transactionAddedToMempool events are processed + promise.set_value(); + SyncWithValidationInterfaceQueue(); + BOOST_CHECK_EQUAL(addtx_count, 4); + + + TestUnloadWallet(std::move(wallet)); + + + // Load wallet again, this time creating new block and mempool transactions + // paying to the wallet as the wallet finishes loading and syncing the + // queue so the events have to be handled immediately. Releasing the wallet + // lock during the sync is a little artificial but is needed to avoid a + // deadlock during the sync and simulates a new block notification happening + // as soon as possible. + addtx_count = 0; + auto handler = HandleLoadWallet([&](std::unique_ptr<interfaces::Wallet> wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet->wallet()->cs_wallet) { + BOOST_CHECK(rescan_completed); + m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + block_tx = TestSimpleSpend(*m_coinbase_txns[2], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + m_coinbase_txns.push_back(CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + mempool_tx = TestSimpleSpend(*m_coinbase_txns[3], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + BOOST_CHECK(chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); + LEAVE_CRITICAL_SECTION(wallet->wallet()->cs_wallet); + SyncWithValidationInterfaceQueue(); + ENTER_CRITICAL_SECTION(wallet->wallet()->cs_wallet); + }); + wallet = TestLoadWallet(*chain); + BOOST_CHECK_EQUAL(addtx_count, 4); + { + LOCK(wallet->cs_wallet); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(mempool_tx.GetHash()), 1U); + } + + + TestUnloadWallet(std::move(wallet)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 6d46841cee..7824563254 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -27,6 +27,7 @@ #include <util/fees.h> #include <util/moneystr.h> #include <util/rbf.h> +#include <util/string.h> #include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/fees.h> @@ -149,34 +150,34 @@ void UnloadWallet(std::shared_ptr<CWallet>&& wallet) } } -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::vector<std::string>& warnings) +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) { try { - if (!CWallet::Verify(chain, location, false, error, warnings)) { - error = "Wallet file verification failed: " + error; + if (!CWallet::Verify(chain, location, error, warnings)) { + error = Untranslated("Wallet file verification failed.") + Untranslated(" ") + error; return nullptr; } std::shared_ptr<CWallet> wallet = CWallet::CreateWalletFromFile(chain, location, error, warnings); if (!wallet) { - error = "Wallet loading failed: " + error; + error = Untranslated("Wallet loading failed.") + Untranslated(" ") + error; return nullptr; } AddWallet(wallet); wallet->postInitProcess(); return wallet; } catch (const std::runtime_error& e) { - error = e.what(); + error = Untranslated(e.what()); return nullptr; } } -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::vector<std::string>& warnings) +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) { return LoadWallet(chain, WalletLocation(name), error, warnings); } -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result) +WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result) { // Indicate that the wallet is actually supposed to be blank and not just blank to make it encrypted bool create_blank = (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET); @@ -189,39 +190,39 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& // Check the wallet file location WalletLocation location(name); if (location.Exists()) { - error = "Wallet " + location.GetName() + " already exists."; + error = strprintf(Untranslated("Wallet %s already exists."), location.GetName()); return WalletCreationStatus::CREATION_FAILED; } // Wallet::Verify will check if we're trying to create a wallet with a duplicate name. - if (!CWallet::Verify(chain, location, false, error, warnings)) { - error = "Wallet file verification failed: " + error; + if (!CWallet::Verify(chain, location, error, warnings)) { + error = Untranslated("Wallet file verification failed.") + Untranslated(" ") + error; return WalletCreationStatus::CREATION_FAILED; } // Do not allow a passphrase when private keys are disabled if (!passphrase.empty() && (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - error = "Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled."; + error = Untranslated("Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled."); return WalletCreationStatus::CREATION_FAILED; } // Make the wallet std::shared_ptr<CWallet> wallet = CWallet::CreateWalletFromFile(chain, location, error, warnings, wallet_creation_flags); if (!wallet) { - error = "Wallet creation failed: " + error; + error = Untranslated("Wallet creation failed.") + Untranslated(" ") + error; return WalletCreationStatus::CREATION_FAILED; } // Encrypt the wallet if (!passphrase.empty() && !(wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { if (!wallet->EncryptWallet(passphrase)) { - error = "Error: Wallet created but failed to encrypt."; + error = Untranslated("Error: Wallet created but failed to encrypt."); return WalletCreationStatus::ENCRYPTION_FAILED; } if (!create_blank) { // Unlock the wallet if (!wallet->Unlock(passphrase)) { - error = "Error: Wallet was encrypted but could not be unlocked"; + error = Untranslated("Error: Wallet was encrypted but could not be unlocked"); return WalletCreationStatus::ENCRYPTION_FAILED; } @@ -233,7 +234,7 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& } else { for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { if (!spk_man->SetupGeneration()) { - error = "Unable to generate initial keys"; + error = Untranslated("Unable to generate initial keys"); return WalletCreationStatus::CREATION_FAILED; } } @@ -783,19 +784,19 @@ bool CWallet::IsSpentKey(const uint256& hash, unsigned int n) const return false; } -bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) +CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx, bool fFlushOnClose) { LOCK(cs_wallet); WalletBatch batch(*database, "r+", fFlushOnClose); - uint256 hash = wtxIn.GetHash(); + uint256 hash = tx->GetHash(); if (IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE)) { // Mark used destinations std::set<CTxDestination> tx_destinations; - for (const CTxIn& txin : wtxIn.tx->vin) { + for (const CTxIn& txin : tx->vin) { const COutPoint& op = txin.prevout; SetSpentKeyState(batch, op.hash, op.n, true, tx_destinations); } @@ -804,11 +805,12 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) } // Inserts only if not already there, returns tx inserted or tx found - std::pair<std::map<uint256, CWalletTx>::iterator, bool> ret = mapWallet.insert(std::make_pair(hash, wtxIn)); + auto ret = mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(hash), std::forward_as_tuple(this, tx)); CWalletTx& wtx = (*ret.first).second; - wtx.BindWallet(this); bool fInsertedNew = ret.second; + bool fUpdated = update_wtx && update_wtx(wtx, fInsertedNew); if (fInsertedNew) { + wtx.m_confirm = confirm; wtx.nTimeReceived = chain().getAdjustedTime(); wtx.nOrderPos = IncOrderPosNext(&batch); wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -816,43 +818,37 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) AddToSpends(hash); } - bool fUpdated = false; if (!fInsertedNew) { - if (wtxIn.m_confirm.status != wtx.m_confirm.status) { - wtx.m_confirm.status = wtxIn.m_confirm.status; - wtx.m_confirm.nIndex = wtxIn.m_confirm.nIndex; - wtx.m_confirm.hashBlock = wtxIn.m_confirm.hashBlock; - wtx.m_confirm.block_height = wtxIn.m_confirm.block_height; + if (confirm.status != wtx.m_confirm.status) { + wtx.m_confirm.status = confirm.status; + wtx.m_confirm.nIndex = confirm.nIndex; + wtx.m_confirm.hashBlock = confirm.hashBlock; + wtx.m_confirm.block_height = confirm.block_height; fUpdated = true; } else { - assert(wtx.m_confirm.nIndex == wtxIn.m_confirm.nIndex); - assert(wtx.m_confirm.hashBlock == wtxIn.m_confirm.hashBlock); - assert(wtx.m_confirm.block_height == wtxIn.m_confirm.block_height); - } - if (wtxIn.fFromMe && wtxIn.fFromMe != wtx.fFromMe) - { - wtx.fFromMe = wtxIn.fFromMe; - fUpdated = true; + assert(wtx.m_confirm.nIndex == confirm.nIndex); + assert(wtx.m_confirm.hashBlock == confirm.hashBlock); + assert(wtx.m_confirm.block_height == confirm.block_height); } // If we have a witness-stripped version of this transaction, and we // see a new version with a witness, then we must be upgrading a pre-segwit // wallet. Store the new version of the transaction with the witness, // as the stripped-version must be invalid. // TODO: Store all versions of the transaction, instead of just one. - if (wtxIn.tx->HasWitness() && !wtx.tx->HasWitness()) { - wtx.SetTx(wtxIn.tx); + if (tx->HasWitness() && !wtx.tx->HasWitness()) { + wtx.SetTx(tx); fUpdated = true; } } //// debug print - WalletLogPrintf("AddToWallet %s %s%s\n", wtxIn.GetHash().ToString(), (fInsertedNew ? "new" : ""), (fUpdated ? "update" : "")); + WalletLogPrintf("AddToWallet %s %s%s\n", hash.ToString(), (fInsertedNew ? "new" : ""), (fUpdated ? "update" : "")); // Write to disk if (fInsertedNew || fUpdated) if (!batch.WriteTx(wtx)) - return false; + return nullptr; // Break debit/credit balance caches: wtx.MarkDirty(); @@ -866,7 +862,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) if (!strCmd.empty()) { - boost::replace_all(strCmd, "%s", wtxIn.GetHash().GetHex()); + boost::replace_all(strCmd, "%s", hash.GetHex()); #ifndef WIN32 // Substituting the wallet name isn't currently supported on windows // because windows shell escaping has not been implemented yet: @@ -880,36 +876,36 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) } #endif - return true; + return &wtx; } -void CWallet::LoadToWallet(CWalletTx& wtxIn) +bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx) { - // If wallet doesn't have a chain (e.g bitcoin-wallet), lock can't be taken. - auto locked_chain = LockChain(); - if (locked_chain) { - Optional<int> block_height = locked_chain->getBlockHeight(wtxIn.m_confirm.hashBlock); + const auto& ins = mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(hash), std::forward_as_tuple(this, nullptr)); + CWalletTx& wtx = ins.first->second; + if (!fill_wtx(wtx, ins.second)) { + return false; + } + // If wallet doesn't have a chain (e.g wallet-tool), don't bother to update txn. + if (HaveChain()) { + Optional<int> block_height = chain().getBlockHeight(wtx.m_confirm.hashBlock); if (block_height) { // Update cached block height variable since it not stored in the // serialized transaction. - wtxIn.m_confirm.block_height = *block_height; - } else if (wtxIn.isConflicted() || wtxIn.isConfirmed()) { + wtx.m_confirm.block_height = *block_height; + } else if (wtx.isConflicted() || wtx.isConfirmed()) { // If tx block (or conflicting block) was reorged out of chain // while the wallet was shutdown, change tx status to UNCONFIRMED // and reset block height, hash, and index. ABANDONED tx don't have // associated blocks and don't need to be updated. The case where a // transaction was reorged out while online and then reconfirmed // while offline is covered by the rescan logic. - wtxIn.setUnconfirmed(); - wtxIn.m_confirm.hashBlock = uint256(); - wtxIn.m_confirm.block_height = 0; - wtxIn.m_confirm.nIndex = 0; + wtx.setUnconfirmed(); + wtx.m_confirm.hashBlock = uint256(); + wtx.m_confirm.block_height = 0; + wtx.m_confirm.nIndex = 0; } } - uint256 hash = wtxIn.GetHash(); - const auto& ins = mapWallet.emplace(hash, wtxIn); - CWalletTx& wtx = ins.first->second; - wtx.BindWallet(this); if (/* insertion took place */ ins.second) { wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); } @@ -923,6 +919,7 @@ void CWallet::LoadToWallet(CWalletTx& wtxIn) } } } + return true; } bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool fUpdate) @@ -961,13 +958,9 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Co } } - CWalletTx wtx(this, ptx); - // Block disconnection override an abandoned tx as unconfirmed // which means user may have to call abandontransaction again - wtx.m_confirm = confirm; - - return AddToWallet(wtx, false); + return AddToWallet(MakeTransactionRef(tx), confirm, /* update_wtx= */ nullptr, /* fFlushOnClose= */ false); } } return false; @@ -975,7 +968,6 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Co bool CWallet::TransactionCanBeAbandoned(const uint256& hashTx) const { - auto locked_chain = chain().lock(); LOCK(cs_wallet); const CWalletTx* wtx = GetWalletTx(hashTx); return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain() == 0 && !wtx->InMempool(); @@ -993,7 +985,6 @@ void CWallet::MarkInputsDirty(const CTransactionRef& tx) bool CWallet::AbandonTransaction(const uint256& hashTx) { - auto locked_chain = chain().lock(); // Temporary. Removed in upcoming lock cleanup LOCK(cs_wallet); WalletBatch batch(*database, "r+"); @@ -1048,7 +1039,6 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); int conflictconfirms = (m_last_block_processed_height - conflicting_height + 1) * -1; @@ -1111,7 +1101,6 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmatio } void CWallet::transactionAddedToMempool(const CTransactionRef& ptx) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); SyncTransaction(ptx, confirm); @@ -1133,7 +1122,6 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef &ptx) { void CWallet::blockConnected(const CBlock& block, int height) { const uint256& block_hash = block.GetHash(); - auto locked_chain = chain().lock(); LOCK(cs_wallet); m_last_block_processed_height = height; @@ -1147,7 +1135,6 @@ void CWallet::blockConnected(const CBlock& block, int height) void CWallet::blockDisconnected(const CBlock& block, int height) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); // At block disconnection, this will change an abandoned transaction to @@ -1686,7 +1673,6 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc uint256 next_block_hash; bool reorg = false; if (chain().findBlock(block_hash, FoundBlock().data(block)) && !block.IsNull()) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); next_block = chain().findNextBlock(block_hash, block_height, FoundBlock().hash(next_block_hash), &reorg); if (reorg) { @@ -1715,7 +1701,6 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } { - auto locked_chain = chain().lock(); if (!next_block || reorg) { // break successfully when rescan has reached the tip, or // previous block is no longer on the chain due to a reorg @@ -1928,16 +1913,16 @@ bool CWalletTx::InMempool() const return fInMempool; } -bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain) const +bool CWalletTx::IsTrusted() const { std::set<uint256> s; - return IsTrusted(locked_chain, s); + return IsTrusted(s); } -bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const +bool CWalletTx::IsTrusted(std::set<uint256>& trusted_parents) const { // Quick answer in most cases - if (!locked_chain.checkFinalTx(*tx)) return false; + if (!pwallet->chain().checkFinalTx(*tx)) return false; int nDepth = GetDepthInMainChain(); if (nDepth >= 1) return true; if (nDepth < 0) return false; @@ -1959,7 +1944,7 @@ bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint25 // If we've already trusted this parent, continue if (trusted_parents.count(parent->GetHash())) continue; // Recurse to check that the parent is also trusted - if (!parent->IsTrusted(locked_chain, trusted_parents)) return false; + if (!parent->IsTrusted(trusted_parents)) return false; trusted_parents.insert(parent->GetHash()); } return true; @@ -1997,14 +1982,9 @@ void CWallet::ResendWalletTransactions() nNextResend = GetTime() + (12 * 60 * 60) + GetRand(24 * 60 * 60); if (fFirst) return; - // Only do it if there's been a new block since last time - if (m_best_block_time < nLastResend) return; - nLastResend = GetTime(); - int submitted_tx_count = 0; - { // locked_chain and cs_wallet scope - auto locked_chain = chain().lock(); + { // cs_wallet scope LOCK(cs_wallet); // Relay transactions @@ -2017,7 +1997,7 @@ void CWallet::ResendWalletTransactions() std::string unused_err_string; if (wtx.SubmitMemoryPoolAndRelay(unused_err_string, true)) ++submitted_tx_count; } - } // locked_chain and cs_wallet + } // cs_wallet if (submitted_tx_count > 0) { WalletLogPrintf("%s: resubmit %u unconfirmed transactions\n", __func__, submitted_tx_count); @@ -2045,13 +2025,12 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons Balance ret; isminefilter reuse_filter = avoid_reuse ? ISMINE_NO : ISMINE_USED; { - auto locked_chain = chain().lock(); LOCK(cs_wallet); std::set<uint256> trusted_parents; for (const auto& entry : mapWallet) { const CWalletTx& wtx = entry.second; - const bool is_trusted{wtx.IsTrusted(*locked_chain, trusted_parents)}; + const bool is_trusted{wtx.IsTrusted(trusted_parents)}; const int tx_depth{wtx.GetDepthInMainChain()}; const CAmount tx_credit_mine{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; @@ -2072,12 +2051,11 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const { - auto locked_chain = chain().lock(); LOCK(cs_wallet); CAmount balance = 0; std::vector<COutput> vCoins; - AvailableCoins(*locked_chain, vCoins, true, coinControl); + AvailableCoins(vCoins, true, coinControl); for (const COutput& out : vCoins) { if (out.fSpendable) { balance += out.tx->tx->vout[out.i].nValue; @@ -2086,7 +2064,7 @@ CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const return balance; } -void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<COutput>& vCoins, bool fOnlySafe, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount) const +void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount) const { AssertLockHeld(cs_wallet); @@ -2104,7 +2082,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< const uint256& wtxid = entry.first; const CWalletTx& wtx = entry.second; - if (!locked_chain.checkFinalTx(*wtx.tx)) { + if (!chain().checkFinalTx(*wtx.tx)) { continue; } @@ -2120,7 +2098,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = wtx.IsTrusted(locked_chain, trusted_parents); + bool safeTx = wtx.IsTrusted(trusted_parents); // We should not consider coins from transactions that are replacing // other transactions. @@ -2208,14 +2186,14 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< } } -std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins(interfaces::Chain::Lock& locked_chain) const +std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins() const { AssertLockHeld(cs_wallet); std::map<CTxDestination, std::vector<COutput>> result; std::vector<COutput> availableCoins; - AvailableCoins(locked_chain, availableCoins); + AvailableCoins(availableCoins); for (const COutput& coin : availableCoins) { CTxDestination address; @@ -2532,7 +2510,7 @@ SigningResult CWallet::SignMessage(const std::string& message, const PKHash& pkh return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; } -bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl coinControl) +bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl coinControl) { std::vector<CRecipient> vecSend; @@ -2551,11 +2529,10 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC // Acquire the locks to prevent races to the new locked unspents between the // CreateTransaction call and LockCoin calls (when lockUnspents is true). - auto locked_chain = chain().lock(); LOCK(cs_wallet); CTransactionRef tx_new; - if (!CreateTransaction(*locked_chain, vecSend, tx_new, nFeeRet, nChangePosInOut, strFailReason, coinControl, false)) { + if (!CreateTransaction(vecSend, tx_new, nFeeRet, nChangePosInOut, error, coinControl, false)) { return false; } @@ -2671,8 +2648,7 @@ OutputType CWallet::TransactionChangeType(OutputType change_type, const std::vec return m_default_address_type; } -bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, - int& nChangePosInOut, std::string& strFailReason, const CCoinControl& coin_control, bool sign) +bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, bool sign) { CAmount nValue = 0; const OutputType change_type = TransactionChangeType(coin_control.m_change_type ? *coin_control.m_change_type : m_default_change_type, vecSend); @@ -2683,7 +2659,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std { if (nValue < 0 || recipient.nAmount < 0) { - strFailReason = _("Transaction amounts must not be negative").translated; + error = _("Transaction amounts must not be negative"); return false; } nValue += recipient.nAmount; @@ -2693,7 +2669,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } if (vecSend.empty()) { - strFailReason = _("Transaction must have at least one recipient").translated; + error = _("Transaction must have at least one recipient"); return false; } @@ -2703,12 +2679,11 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std int nBytes; { std::set<CInputCoin> setCoins; - auto locked_chain = chain().lock(); LOCK(cs_wallet); txNew.nLockTime = GetLocktimeForNewTransaction(chain(), GetLastBlockHash(), GetLastBlockHeight()); { std::vector<COutput> vAvailableCoins; - AvailableCoins(*locked_chain, vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); + AvailableCoins(vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); CoinSelectionParams coin_selection_params; // Parameters for coin selection, init with dummy // Create change script that will be used if we need change @@ -2731,10 +2706,13 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // destination in case we don't need change. CTxDestination dest; if (!reservedest.GetReservedDestination(dest, true)) { - strFailReason = _("Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.").translated; + error = _("Transaction needs a change address, but we can't generate it. Please call keypoolrefill first."); } scriptChange = GetScriptForDestination(dest); - assert(!dest.empty() || scriptChange.empty()); + // A valid destination implies a change script (and + // vice-versa). An empty change script will abort later, if the + // change keypool ran out, but change is required. + CHECK_NONFATAL(IsValidDestination(dest) != scriptChange.empty()); } CTxOut change_prototype_txout(0, scriptChange); coin_selection_params.change_output_size = GetSerializeSize(change_prototype_txout); @@ -2793,12 +2771,12 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std if (recipient.fSubtractFeeFromAmount && nFeeRet > 0) { if (txout.nValue < 0) - strFailReason = _("The transaction amount is too small to pay the fee").translated; + error = _("The transaction amount is too small to pay the fee"); else - strFailReason = _("The transaction amount is too small to send after the fee has been deducted").translated; + error = _("The transaction amount is too small to send after the fee has been deducted"); } else - strFailReason = _("Transaction amount too small").translated; + error = _("Transaction amount too small"); return false; } txNew.vout.push_back(txout); @@ -2826,7 +2804,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std continue; } else { - strFailReason = _("Insufficient funds").translated; + error = _("Insufficient funds"); return false; } } @@ -2857,7 +2835,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } else if ((unsigned int)nChangePosInOut > txNew.vout.size()) { - strFailReason = _("Change index out of range").translated; + error = _("Change index out of range"); return false; } @@ -2876,14 +2854,14 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std nBytes = CalculateMaximumSignedTxSize(CTransaction(txNew), this, coin_control.fAllowWatchOnly); if (nBytes < 0) { - strFailReason = _("Signing transaction failed").translated; + error = _("Signing transaction failed"); return false; } nFeeNeeded = GetMinimumFee(*this, nBytes, coin_control, &feeCalc); if (feeCalc.reason == FeeReason::FALLBACK && !m_allow_fallback_fee) { // eventually allow a fallback fee - strFailReason = _("Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee.").translated; + error = _("Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee."); return false; } @@ -2923,7 +2901,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // fee to pay for the new output and still meet nFeeNeeded // Or we should have just subtracted fee from recipients and // nFeeNeeded should not have changed - strFailReason = _("Transaction fee and change calculation failed").translated; + error = _("Transaction fee and change calculation failed"); return false; } @@ -2951,7 +2929,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std continue; } - // Give up if change keypool ran out and we failed to find a solution without change: + // Give up if change keypool ran out and change is required if (scriptChange.empty() && nChangePosInOut != -1) { return false; } @@ -2976,7 +2954,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } if (sign && !SignTransaction(txNew)) { - strFailReason = _("Signing transaction failed").translated; + error = _("Signing transaction failed"); return false; } @@ -2986,20 +2964,20 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // Limit size if (GetTransactionWeight(*tx) > MAX_STANDARD_TX_WEIGHT) { - strFailReason = _("Transaction too large").translated; + error = _("Transaction too large"); return false; } } if (nFeeRet > m_default_max_tx_fee) { - strFailReason = TransactionErrorString(TransactionError::MAX_FEE_EXCEEDED); + error = Untranslated(TransactionErrorString(TransactionError::MAX_FEE_EXCEEDED)); return false; } if (gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS)) { // Lastly, ensure this tx will pass the mempool's chain limits if (!chain().checkChainLimits(tx)) { - strFailReason = _("Transaction has too long of a mempool chain").translated; + error = _("Transaction has too long of a mempool chain"); return false; } } @@ -3021,31 +2999,31 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm) { - auto locked_chain = chain().lock(); LOCK(cs_wallet); - - CWalletTx wtxNew(this, std::move(tx)); - wtxNew.mapValue = std::move(mapValue); - wtxNew.vOrderForm = std::move(orderForm); - wtxNew.fTimeReceivedIsTxTime = true; - wtxNew.fFromMe = true; - - WalletLogPrintf("CommitTransaction:\n%s", wtxNew.tx->ToString()); /* Continued */ + WalletLogPrintf("CommitTransaction:\n%s", tx->ToString()); /* Continued */ // Add tx to wallet, because if it has change it's also ours, // otherwise just for transaction history. - AddToWallet(wtxNew); + AddToWallet(tx, {}, [&](CWalletTx& wtx, bool new_tx) { + CHECK_NONFATAL(wtx.mapValue.empty()); + CHECK_NONFATAL(wtx.vOrderForm.empty()); + wtx.mapValue = std::move(mapValue); + wtx.vOrderForm = std::move(orderForm); + wtx.fTimeReceivedIsTxTime = true; + wtx.fFromMe = true; + return true; + }); // Notify that old coins are spent - for (const CTxIn& txin : wtxNew.tx->vin) { + for (const CTxIn& txin : tx->vin) { CWalletTx &coin = mapWallet.at(txin.prevout.hash); - coin.BindWallet(this); + coin.MarkDirty(); NotifyTransactionChanged(this, coin.GetHash(), CT_UPDATED); } // Get the inserted-CWalletTx from mapWallet so that the // fInMempool flag is cached properly - CWalletTx& wtx = mapWallet.at(wtxNew.GetHash()); + CWalletTx& wtx = mapWallet.at(tx->GetHash()); if (!fBroadcastTransactions) { // Don't submit tx to the mempool @@ -3061,11 +3039,6 @@ void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { - // Even if we don't use this lock in this function, we want to preserve - // lock order in LoadToWallet if query of chain state is needed to know - // tx status. If lock can't be taken (e.g bitcoin-wallet), tx confirmation - // status may be not reliable. - auto locked_chain = LockChain(); LOCK(cs_wallet); fFirstRunRet = false; @@ -3122,7 +3095,7 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 return DBErrors::LOAD_OK; } -DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) +DBErrors CWallet::ZapWalletTx(std::list<CWalletTx>& vWtx) { DBErrors nZapWalletTxRet = WalletBatch(*database,"cr+").ZapWalletTx(vWtx); if (nZapWalletTxRet == DBErrors::NEED_REWRITE) @@ -3284,7 +3257,7 @@ void CWallet::MarkDestinationsDirty(const std::set<CTxDestination>& destinations } } -std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain::Lock& locked_chain) const +std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() const { std::map<CTxDestination, CAmount> balances; @@ -3295,7 +3268,7 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: { const CWalletTx& wtx = walletEntry.second; - if (!wtx.IsTrusted(locked_chain, trusted_parents)) + if (!wtx.IsTrusted(trusted_parents)) continue; if (wtx.IsImmatureCoinBase()) @@ -3511,7 +3484,7 @@ void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const /** @} */ // end of Actions -void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<CKeyID, int64_t>& mapKeyBirth) const { +void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { AssertLockHeld(cs_wallet); mapKeyBirth.clear(); @@ -3677,7 +3650,7 @@ std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const return values; } -bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, std::string& error_string, std::vector<std::string>& warnings) +bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error_string, std::vector<bilingual_str>& warnings) { // Do some checking on wallet path. It should be either a: // @@ -3691,17 +3664,17 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b if (!(path_type == fs::file_not_found || path_type == fs::directory_file || (path_type == fs::symlink_file && fs::is_directory(wallet_path)) || (path_type == fs::regular_file && fs::path(location.GetName()).filename() == location.GetName()))) { - error_string = strprintf( + error_string = Untranslated(strprintf( "Invalid -wallet path '%s'. -wallet path should point to a directory where wallet.dat and " "database/log.?????????? files can be stored, a location where such a directory could be created, " "or (for backwards compatibility) the name of an existing data file in -walletdir (%s)", - location.GetName(), GetWalletDir()); + location.GetName(), GetWalletDir())); return false; } // Make sure that the wallet path doesn't clash with an existing wallet path if (IsWalletLoaded(wallet_path)) { - error_string = strprintf("Error loading wallet %s. Duplicate -wallet filename specified.", location.GetName()); + error_string = Untranslated(strprintf("Error loading wallet %s. Duplicate -wallet filename specified.", location.GetName())); return false; } @@ -3713,33 +3686,19 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b return false; } } catch (const fs::filesystem_error& e) { - error_string = strprintf("Error loading wallet %s. %s", location.GetName(), fsbridge::get_filesystem_error_message(e)); + error_string = Untranslated(strprintf("Error loading wallet %s. %s", location.GetName(), fsbridge::get_filesystem_error_message(e))); return false; } - if (salvage_wallet) { - // Recover readable keypairs: - CWallet dummyWallet(&chain, WalletLocation(), WalletDatabase::CreateDummy()); - std::string backup_filename; - // Even if we don't use this lock in this function, we want to preserve - // lock order in LoadToWallet if query of chain state is needed to know - // tx status. If lock can't be taken, tx confirmation status may be not - // reliable. - auto locked_chain = dummyWallet.LockChain(); - if (!WalletBatch::Recover(wallet_path, (void *)&dummyWallet, WalletBatch::RecoverKeysOnlyFilter, backup_filename)) { - return false; - } - } - - return WalletBatch::VerifyDatabaseFile(wallet_path, warnings, error_string); + return WalletBatch::VerifyDatabaseFile(wallet_path, error_string); } -std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::vector<std::string>& warnings, uint64_t wallet_creation_flags) +std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags) { const std::string walletFile = WalletDataFilePath(location.GetPath()).string(); // needed to restore wallet transaction meta data after -zapwallettxes - std::vector<CWalletTx> vWtx; + std::list<CWalletTx> vWtx; if (gArgs.GetBoolArg("-zapwallettxes", false)) { chain.initMessage(_("Zapping all transactions from wallet...").translated); @@ -3747,7 +3706,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(&chain, location, WalletDatabase::Create(location.GetPath())); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DBErrors::LOAD_OK) { - error = strprintf(_("Error loading %s: Wallet corrupted").translated, walletFile); + error = strprintf(_("Error loading %s: Wallet corrupted"), walletFile); return nullptr; } } @@ -3762,26 +3721,26 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun); if (nLoadWalletRet != DBErrors::LOAD_OK) { if (nLoadWalletRet == DBErrors::CORRUPT) { - error = strprintf(_("Error loading %s: Wallet corrupted").translated, walletFile); + error = strprintf(_("Error loading %s: Wallet corrupted"), walletFile); return nullptr; } else if (nLoadWalletRet == DBErrors::NONCRITICAL_ERROR) { warnings.push_back(strprintf(_("Error reading %s! All keys read correctly, but transaction data" - " or address book entries might be missing or incorrect.").translated, + " or address book entries might be missing or incorrect."), walletFile)); } else if (nLoadWalletRet == DBErrors::TOO_NEW) { - error = strprintf(_("Error loading %s: Wallet requires newer version of %s").translated, walletFile, PACKAGE_NAME); + error = strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, PACKAGE_NAME); return nullptr; } else if (nLoadWalletRet == DBErrors::NEED_REWRITE) { - error = strprintf(_("Wallet needed to be rewritten: restart %s to complete").translated, PACKAGE_NAME); + error = strprintf(_("Wallet needed to be rewritten: restart %s to complete"), PACKAGE_NAME); return nullptr; } else { - error = strprintf(_("Error loading %s").translated, walletFile); + error = strprintf(_("Error loading %s"), walletFile); return nullptr; } } @@ -3807,47 +3766,46 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // Legacy wallets need SetupGeneration here. for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { if (!spk_man->SetupGeneration()) { - error = _("Unable to generate initial keys").translated; + error = _("Unable to generate initial keys"); return nullptr; } } } } - auto locked_chain = chain.lock(); - walletInstance->chainStateFlushed(locked_chain->getTipLocator()); + walletInstance->chainStateFlushed(chain.getTipLocator()); } else if (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS) { // Make it impossible to disable private keys after creation - error = strprintf(_("Error loading %s: Private keys can only be disabled during creation").translated, walletFile); + error = strprintf(_("Error loading %s: Private keys can only be disabled during creation"), walletFile); return NULL; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { if (spk_man->HavePrivateKeys()) { - warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys").translated, walletFile)); + warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys"), walletFile)); break; } } } if (!gArgs.GetArg("-addresstype", "").empty() && !ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { - error = strprintf(_("Unknown address type '%s'").translated, gArgs.GetArg("-addresstype", "")); + error = strprintf(_("Unknown address type '%s'"), gArgs.GetArg("-addresstype", "")); return nullptr; } if (!gArgs.GetArg("-changetype", "").empty() && !ParseOutputType(gArgs.GetArg("-changetype", ""), walletInstance->m_default_change_type)) { - error = strprintf(_("Unknown change type '%s'").translated, gArgs.GetArg("-changetype", "")); + error = strprintf(_("Unknown change type '%s'"), gArgs.GetArg("-changetype", "")); return nullptr; } if (gArgs.IsArgSet("-mintxfee")) { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-mintxfee", ""), n) || 0 == n) { - error = AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", "")).translated; + error = AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", "")); return nullptr; } if (n > HIGH_TX_FEE_PER_KB) { - warnings.push_back(AmountHighWarn("-mintxfee").translated + " " + - _("This is the minimum transaction fee you pay on every transaction.").translated); + warnings.push_back(AmountHighWarn("-mintxfee") + Untranslated(" ") + + _("This is the minimum transaction fee you pay on every transaction.")); } walletInstance->m_min_fee = CFeeRate(n); } @@ -3855,12 +3813,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-fallbackfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-fallbackfee", ""), nFeePerK)) { - error = strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'").translated, gArgs.GetArg("-fallbackfee", "")); + error = strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'"), gArgs.GetArg("-fallbackfee", "")); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - warnings.push_back(AmountHighWarn("-fallbackfee").translated + " " + - _("This is the transaction fee you may pay when fee estimates are not available.").translated); + warnings.push_back(AmountHighWarn("-fallbackfee") + Untranslated(" ") + + _("This is the transaction fee you may pay when fee estimates are not available.")); } walletInstance->m_fallback_fee = CFeeRate(nFeePerK); } @@ -3870,28 +3828,28 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-discardfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-discardfee", ""), nFeePerK)) { - error = strprintf(_("Invalid amount for -discardfee=<amount>: '%s'").translated, gArgs.GetArg("-discardfee", "")); + error = strprintf(_("Invalid amount for -discardfee=<amount>: '%s'"), gArgs.GetArg("-discardfee", "")); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - warnings.push_back(AmountHighWarn("-discardfee").translated + " " + - _("This is the transaction fee you may discard if change is smaller than dust at this level").translated); + warnings.push_back(AmountHighWarn("-discardfee") + Untranslated(" ") + + _("This is the transaction fee you may discard if change is smaller than dust at this level")); } walletInstance->m_discard_rate = CFeeRate(nFeePerK); } if (gArgs.IsArgSet("-paytxfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-paytxfee", ""), nFeePerK)) { - error = AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", "")).translated; + error = AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", "")); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - warnings.push_back(AmountHighWarn("-paytxfee").translated + " " + - _("This is the transaction fee you will pay if you send a transaction.").translated); + warnings.push_back(AmountHighWarn("-paytxfee") + Untranslated(" ") + + _("This is the transaction fee you will pay if you send a transaction.")); } walletInstance->m_pay_tx_fee = CFeeRate(nFeePerK, 1000); if (walletInstance->m_pay_tx_fee < chain.relayMinFee()) { - error = strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)").translated, + error = strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)"), gArgs.GetArg("-paytxfee", ""), chain.relayMinFee().ToString()); return nullptr; } @@ -3900,23 +3858,23 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-maxtxfee")) { CAmount nMaxFee = 0; if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) { - error = AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", "")).translated; + error = AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", "")); return nullptr; } if (nMaxFee > HIGH_MAX_TX_FEE) { - warnings.push_back(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.").translated); + warnings.push_back(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); } if (CFeeRate(nMaxFee, 1000) < chain.relayMinFee()) { - error = strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)").translated, - gArgs.GetArg("-maxtxfee", ""), chain.relayMinFee().ToString()); + error = strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), + gArgs.GetArg("-maxtxfee", ""), chain.relayMinFee().ToString()); return nullptr; } walletInstance->m_default_max_tx_fee = nMaxFee; } if (chain.relayMinFee().GetFeePerK() > HIGH_TX_FEE_PER_KB) { - warnings.push_back(AmountHighWarn("-minrelaytxfee").translated + " " + - _("The wallet will avoid paying less than the minimum relay fee.").translated); + warnings.push_back(AmountHighWarn("-minrelaytxfee") + Untranslated(" ") + + _("The wallet will avoid paying less than the minimum relay fee.")); } walletInstance->m_confirm_target = gArgs.GetArg("-txconfirmtarget", DEFAULT_TX_CONFIRM_TARGET); @@ -3928,24 +3886,33 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // Try to top up keypool. No-op if the wallet is locked. walletInstance->TopUpKeyPool(); - auto locked_chain = chain.lock(); LOCK(walletInstance->cs_wallet); + // Register wallet with validationinterface. It's done before rescan to avoid + // missing block connections between end of rescan and validation subscribing. + // Because of wallet lock being hold, block connection notifications are going to + // be pending on the validation-side until lock release. It's likely to have + // block processing duplicata (if rescan block range overlaps with notification one) + // but we guarantee at least than wallet state is correct after notifications delivery. + // This is temporary until rescan and notifications delivery are unified under same + // interface. + walletInstance->m_chain_notifications_handler = walletInstance->chain().handleNotifications(walletInstance); + int rescan_height = 0; if (!gArgs.GetBoolArg("-rescan", false)) { WalletBatch batch(*walletInstance->database); CBlockLocator locator; if (batch.ReadBestBlock(locator)) { - if (const Optional<int> fork_height = locked_chain->findLocatorFork(locator)) { + if (const Optional<int> fork_height = chain.findLocatorFork(locator)) { rescan_height = *fork_height; } } } - const Optional<int> tip_height = locked_chain->getHeight(); + const Optional<int> tip_height = chain.getHeight(); if (tip_height) { - walletInstance->m_last_block_processed = locked_chain->getBlockHash(*tip_height); + walletInstance->m_last_block_processed = chain.getBlockHash(*tip_height); walletInstance->m_last_block_processed_height = *tip_height; } else { walletInstance->m_last_block_processed.SetNull(); @@ -3962,12 +3929,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // If a block is pruned after this check, we will load the wallet, // but fail the rescan with a generic error. int block_height = *tip_height; - while (block_height > 0 && locked_chain->haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { + while (block_height > 0 && chain.haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { --block_height; } if (rescan_height != block_height) { - error = _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)").translated; + error = _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); return nullptr; } } @@ -3984,19 +3951,19 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (!time_first_key || time < *time_first_key) time_first_key = time; } if (time_first_key) { - if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, nullptr)) { + if (Optional<int> first_block = chain.findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, nullptr)) { rescan_height = *first_block; } } { WalletRescanReserver reserver(*walletInstance); - if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { - error = _("Failed to rescan the wallet during initialization").translated; + if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(chain.getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { + error = _("Failed to rescan the wallet during initialization"); return nullptr; } } - walletInstance->chainStateFlushed(locked_chain->getTipLocator()); + walletInstance->chainStateFlushed(chain.getTipLocator()); walletInstance->database->IncrementUpdateCounter(); // Restore wallet transaction metadata after -zapwallettxes=1 @@ -4031,9 +3998,6 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - // Register with the validation interface. It's ok to do this after rescan since we're still holding locked_chain. - walletInstance->m_chain_notifications_handler = walletInstance->chain().handleNotifications(walletInstance); - walletInstance->SetBroadcastTransactions(gArgs.GetBoolArg("-walletbroadcast", DEFAULT_WALLETBROADCAST)); { @@ -4055,7 +4019,7 @@ const CAddressBookData* CWallet::FindAddressBookEntry(const CTxDestination& dest return &address_book_it->second; } -bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings) +bool CWallet::UpgradeWallet(int version, bilingual_str& error, std::vector<bilingual_str>& warnings) { int prev_version = GetVersion(); int nMaxVersion = version; @@ -4064,12 +4028,12 @@ bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::st WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); nMaxVersion = FEATURE_LATEST; SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately - } - else + } else { WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); + } if (nMaxVersion < GetVersion()) { - error = _("Cannot downgrade wallet").translated; + error = _("Cannot downgrade wallet"); return false; } SetMaxVersion(nMaxVersion); @@ -4079,7 +4043,7 @@ bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::st // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT int max_version = GetVersion(); if (!CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { - error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified.").translated; + error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified."); return false; } @@ -4093,7 +4057,6 @@ bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::st void CWallet::postInitProcess() { - auto locked_chain = chain().lock(); LOCK(cs_wallet); // Add wallet transactions that aren't already in a block to mempool @@ -4390,7 +4353,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() for (bool internal : {false, true}) { for (OutputType t : OUTPUT_TYPES) { - auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, t, internal)); + auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, internal)); if (IsCrypted()) { if (IsLocked()) { throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); @@ -4399,7 +4362,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); } } - spk_manager->SetupDescriptorGeneration(master_key); + spk_manager->SetupDescriptorGeneration(master_key, t); uint256 id = spk_manager->GetID(); m_spk_managers[id] = std::move(spk_manager); SetActiveScriptPubKeyMan(id, t, internal); @@ -4412,7 +4375,7 @@ void CWallet::SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool interna WalletLogPrintf("Setting spkMan to active: id = %s, type = %d, internal = %d\n", id.ToString(), static_cast<int>(type), static_cast<int>(internal)); auto& spk_mans = internal ? m_internal_spk_managers : m_external_spk_managers; auto spk_man = m_spk_managers.at(id).get(); - spk_man->SetType(type, internal); + spk_man->SetInternal(internal); spk_mans[type] = spk_man; if (!memonly) { diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 24c78fceb3..e3141baef0 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -40,6 +40,8 @@ using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; +struct bilingual_str; + //! Explicitly unload and delete the wallet. //! Blocks the current thread after signaling the unload intent so that all //! wallet clients release the wallet. @@ -52,7 +54,7 @@ bool RemoveWallet(const std::shared_ptr<CWallet>& wallet); bool HasWallets(); std::vector<std::shared_ptr<CWallet>> GetWallets(); std::shared_ptr<CWallet> GetWallet(const std::string& name); -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::vector<std::string>& warnings); +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings); std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet); enum class WalletCreationStatus { @@ -61,7 +63,7 @@ enum class WalletCreationStatus { ENCRYPTION_FAILED }; -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result); +WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result); //! -paytxfee default constexpr CAmount DEFAULT_PAY_TX_FEE = 0; @@ -338,15 +340,15 @@ public: mutable bool fInMempool; mutable CAmount nChangeCached; - CWalletTx(const CWallet* pwalletIn, CTransactionRef arg) - : tx(std::move(arg)) + CWalletTx(const CWallet* wallet, CTransactionRef arg) + : pwallet(wallet), + tx(std::move(arg)) { - Init(pwalletIn); + Init(); } - void Init(const CWallet* pwalletIn) + void Init() { - pwallet = pwalletIn; mapValue.clear(); vOrderForm.clear(); fTimeReceivedIsTxTime = false; @@ -414,7 +416,7 @@ public: template<typename Stream> void Unserialize(Stream& s) { - Init(nullptr); + Init(); std::vector<uint256> dummy_vector1; //!< Used to be vMerkleBranch std::vector<CMerkleTx> dummy_vector2; //!< Used to be vtxPrev @@ -463,12 +465,6 @@ public: m_is_cache_empty = true; } - void BindWallet(CWallet *pwalletIn) - { - pwallet = pwalletIn; - MarkDirty(); - } - //! filter decides which addresses will count towards the debit CAmount GetDebit(const isminefilter& filter) const; CAmount GetCredit(const isminefilter& filter) const; @@ -499,8 +495,8 @@ public: bool IsEquivalentTo(const CWalletTx& tx) const; bool InMempool() const; - bool IsTrusted(interfaces::Chain::Lock& locked_chain) const; - bool IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const; + bool IsTrusted() const; + bool IsTrusted(std::set<uint256>& trusted_parents) const; int64_t GetTxTime() const; @@ -553,6 +549,12 @@ public: const uint256& GetHash() const { return tx->GetHash(); } bool IsCoinBase() const { return tx->IsCoinBase(); } bool IsImmatureCoinBase() const; + + // Disable copying of CWalletTx objects to prevent bugs where instances get + // copied in and out of the mapWallet map, and fields are updated in the + // wrong copy. + CWalletTx(CWalletTx const &) = delete; + void operator=(CWalletTx const &x) = delete; }; class COutput @@ -629,7 +631,6 @@ private: std::atomic<bool> fScanningWallet{false}; // controlled by WalletRescanReserver std::atomic<int64_t> m_scanning_start{0}; std::atomic<double> m_scanning_progress{0}; - std::mutex mutexScanning; friend class WalletRescanReserver; //! the current wallet version: clients below this version are not able to load the wallet @@ -639,7 +640,6 @@ private: int nWalletMaxVersion GUARDED_BY(cs_wallet) = FEATURE_BASE; int64_t nNextResend = 0; - int64_t nLastResend = 0; bool fBroadcastTransactions = false; // Local time that the tip block was received. Used to schedule wallet rebroadcasts. std::atomic<int64_t> m_best_block_time {0}; @@ -775,8 +775,8 @@ public: bool IsLocked() const override; bool Lock(); - /** Interface to assert chain access and if successful lock it */ - std::unique_ptr<interfaces::Chain::Lock> LockChain() { return m_chain ? m_chain->lock() : nullptr; } + /** Interface to assert chain access */ + bool HaveChain() const { return m_chain ? true : false; } std::map<uint256, CWalletTx> mapWallet GUARDED_BY(cs_wallet); @@ -805,12 +805,12 @@ public: /** * populate vCoins with vector of available COutputs. */ - void AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector<COutput>& vCoins, bool fOnlySafe = true, const CCoinControl* coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe = true, const CCoinControl* coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Return list of available coins and locked coins grouped by non-change output address. */ - std::map<CTxDestination, std::vector<COutput>> ListCoins(interfaces::Chain::Lock& locked_chain) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + std::map<CTxDestination, std::vector<COutput>> ListCoins() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Find non-change parent output. @@ -869,13 +869,15 @@ public: std::vector<std::string> GetDestValues(const std::string& prefix) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Holds a timestamp at which point the wallet is scheduled (externally) to be relocked. Caller must arrange for actual relocking to occur via Lock(). - int64_t nRelockTime = 0; + int64_t nRelockTime GUARDED_BY(cs_wallet){0}; + // Used to prevent concurrent calls to walletpassphrase RPC. + Mutex m_unlock_mutex; bool Unlock(const SecureString& strWalletPassphrase, bool accept_no_keys = false); bool ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, const SecureString& strNewWalletPassphrase); bool EncryptWallet(const SecureString& strWalletPassphrase); - void GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<CKeyID, int64_t> &mapKeyBirth) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void GetKeyBirthTimes(std::map<CKeyID, int64_t> &mapKeyBirth) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); unsigned int ComputeTimeSmart(const CWalletTx& wtx) const; /** @@ -886,8 +888,17 @@ public: DBErrors ReorderTransactions(); void MarkDirty(); - bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); - void LoadToWallet(CWalletTx& wtxIn) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + //! Callback for updating transaction metadata in mapWallet. + //! + //! @param wtx - reference to mapWallet transaction to update + //! @param new_tx - true if wtx is newly inserted, false if it previously existed + //! + //! @return true if wtx is changed and needs to be saved to disk, otherwise false + using UpdateWalletTxFn = std::function<bool(CWalletTx& wtx, bool new_tx)>; + + CWalletTx* AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx=nullptr, bool fFlushOnClose=true); + bool LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void transactionAddedToMempool(const CTransactionRef& tx) override; void blockConnected(const CBlock& block, int height) override; void blockDisconnected(const CBlock& block, int height) override; @@ -930,7 +941,7 @@ public: * Insert additional inputs into the transaction by * calling CreateTransaction(); */ - bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); + bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); // Fetch the inputs and sign with SIGHASH_ALL. bool SignTransaction(CMutableTransaction& tx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); // Sign the tx given the input coins and sighash. @@ -961,8 +972,7 @@ public: * selected by SelectCoins(); Also create the change output, when needed * @note passing nChangePosInOut as -1 will result in setting a random position */ - bool CreateTransaction(interfaces::Chain::Lock& locked_chain, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, - std::string& strFailReason, const CCoinControl& coin_control, bool sign = true); + bool CreateTransaction(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, bool sign = true); /** * Submit the transaction to the node's mempool and then relay to peers. * Should be called after CreateTransaction unless you want to abort @@ -1012,7 +1022,7 @@ public: int64_t GetOldestKeyPoolTime() const; std::set<std::set<CTxDestination>> GetAddressGroupings() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::map<CTxDestination, CAmount> GetAddressBalances(interfaces::Chain::Lock& locked_chain) const; + std::map<CTxDestination, CAmount> GetAddressBalances() const; std::set<CTxDestination> GetLabelAddresses(const std::string& label) const; @@ -1049,7 +1059,7 @@ public: void chainStateFlushed(const CBlockLocator& loc) override; DBErrors LoadWallet(bool& fFirstRunRet); - DBErrors ZapWalletTx(std::vector<CWalletTx>& vWtx); + DBErrors ZapWalletTx(std::list<CWalletTx>& vWtx); DBErrors ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool SetAddressBook(const CTxDestination& address, const std::string& strName, const std::string& purpose); @@ -1125,10 +1135,10 @@ public: bool MarkReplaced(const uint256& originalHash, const uint256& newHash); //! Verify wallet naming and perform salvage on the wallet if required - static bool Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, std::string& error_string, std::vector<std::string>& warnings); + static bool Verify(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error_string, std::vector<bilingual_str>& warnings); /* Initializes the wallet, returns a new CWallet instance or a null pointer in case of an error */ - static std::shared_ptr<CWallet> CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::vector<std::string>& warnings, uint64_t wallet_creation_flags = 0); + static std::shared_ptr<CWallet> CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags = 0); /** * Wallet post-init setup @@ -1181,7 +1191,7 @@ public: }; /** Upgrade the wallet */ - bool UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings); + bool UpgradeWallet(int version, bilingual_str& error, std::vector<bilingual_str>& warnings); //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; @@ -1276,13 +1286,11 @@ public: bool reserve() { assert(!m_could_reserve); - std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); - if (m_wallet.fScanningWallet) { + if (m_wallet.fScanningWallet.exchange(true)) { return false; } m_wallet.m_scanning_start = GetTimeMillis(); m_wallet.m_scanning_progress = 0; - m_wallet.fScanningWallet = true; m_could_reserve = true; return true; } @@ -1294,7 +1302,6 @@ public: ~WalletRescanReserver() { - std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); if (m_could_reserve) { m_wallet.fScanningWallet = false; } diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 79316ca3e7..e7adbfea77 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -10,6 +10,7 @@ #include <protocol.h> #include <serialize.h> #include <sync.h> +#include <util/bip32.h> #include <util/system.h> #include <util/time.h> #include <wallet/wallet.h> @@ -115,8 +116,19 @@ bool WalletBatch::WriteCryptedKey(const CPubKey& vchPubKey, return false; } - if (!WriteIC(std::make_pair(DBKeys::CRYPTED_KEY, vchPubKey), vchCryptedSecret, false)) { - return false; + // Compute a checksum of the encrypted key + uint256 checksum = Hash(vchCryptedSecret.begin(), vchCryptedSecret.end()); + + const auto key = std::make_pair(DBKeys::CRYPTED_KEY, vchPubKey); + if (!WriteIC(key, std::make_pair(vchCryptedSecret, checksum), false)) { + // It may already exist, so try writing just the checksum + std::vector<unsigned char> val; + if (!m_batch.Read(key, val)) { + return false; + } + if (!WriteIC(key, std::make_pair(val, checksum), true)) { + return false; + } } EraseIC(std::make_pair(DBKeys::KEY, vchPubKey)); return true; @@ -245,6 +257,7 @@ public: std::map<uint256, DescriptorCache> m_descriptor_caches; std::map<std::pair<uint256, CKeyID>, CKey> m_descriptor_keys; std::map<std::pair<uint256, CKeyID>, std::pair<CPubKey, std::vector<unsigned char>>> m_descriptor_crypt_keys; + std::map<uint160, CHDChain> m_hd_chains; CWalletScanState() { } @@ -272,36 +285,43 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == DBKeys::TX) { uint256 hash; ssKey >> hash; - CWalletTx wtx(nullptr /* pwallet */, MakeTransactionRef()); - ssValue >> wtx; - if (wtx.GetHash() != hash) - return false; + // LoadToWallet call below creates a new CWalletTx that fill_wtx + // callback fills with transaction metadata. + auto fill_wtx = [&](CWalletTx& wtx, bool new_tx) { + assert(new_tx); + ssValue >> wtx; + if (wtx.GetHash() != hash) + return false; - // Undo serialize changes in 31600 - if (31404 <= wtx.fTimeReceivedIsTxTime && wtx.fTimeReceivedIsTxTime <= 31703) - { - if (!ssValue.empty()) - { - char fTmp; - char fUnused; - std::string unused_string; - ssValue >> fTmp >> fUnused >> unused_string; - strErr = strprintf("LoadWallet() upgrading tx ver=%d %d %s", - wtx.fTimeReceivedIsTxTime, fTmp, hash.ToString()); - wtx.fTimeReceivedIsTxTime = fTmp; - } - else + // Undo serialize changes in 31600 + if (31404 <= wtx.fTimeReceivedIsTxTime && wtx.fTimeReceivedIsTxTime <= 31703) { - strErr = strprintf("LoadWallet() repairing tx ver=%d %s", wtx.fTimeReceivedIsTxTime, hash.ToString()); - wtx.fTimeReceivedIsTxTime = 0; + if (!ssValue.empty()) + { + char fTmp; + char fUnused; + std::string unused_string; + ssValue >> fTmp >> fUnused >> unused_string; + strErr = strprintf("LoadWallet() upgrading tx ver=%d %d %s", + wtx.fTimeReceivedIsTxTime, fTmp, hash.ToString()); + wtx.fTimeReceivedIsTxTime = fTmp; + } + else + { + strErr = strprintf("LoadWallet() repairing tx ver=%d %s", wtx.fTimeReceivedIsTxTime, hash.ToString()); + wtx.fTimeReceivedIsTxTime = 0; + } + wss.vWalletUpgrade.push_back(hash); } - wss.vWalletUpgrade.push_back(hash); - } - if (wtx.nOrderPos == -1) - wss.fAnyUnordered = true; + if (wtx.nOrderPos == -1) + wss.fAnyUnordered = true; - pwallet->LoadToWallet(wtx); + return true; + }; + if (!pwallet->LoadToWallet(hash, fill_wtx)) { + return false; + } } else if (strType == DBKeys::WATCHS) { wss.nWatchKeys++; CScript script; @@ -390,9 +410,21 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } std::vector<unsigned char> vchPrivKey; ssValue >> vchPrivKey; + + // Get the checksum and check it + bool checksum_valid = false; + if (!ssValue.eof()) { + uint256 checksum; + ssValue >> checksum; + if ((checksum_valid = Hash(vchPrivKey.begin(), vchPrivKey.end()) != checksum)) { + strErr = "Error reading wallet database: Crypted key corrupt"; + return false; + } + } + wss.nCKeys++; - if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey, checksum_valid)) { strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCryptedKey failed"; return false; @@ -405,6 +437,65 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssValue >> keyMeta; wss.nKeyMeta++; pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); + + // Extract some CHDChain info from this metadata if it has any + if (keyMeta.nVersion >= CKeyMetadata::VERSION_WITH_HDDATA && !keyMeta.hd_seed_id.IsNull() && keyMeta.hdKeypath.size() > 0) { + // Get the path from the key origin or from the path string + // Not applicable when path is "s" as that indicates a seed + bool internal = false; + uint32_t index = 0; + if (keyMeta.hdKeypath != "s") { + std::vector<uint32_t> path; + if (keyMeta.has_key_origin) { + // We have a key origin, so pull it from its path vector + path = keyMeta.key_origin.path; + } else { + // No key origin, have to parse the string + if (!ParseHDKeypath(keyMeta.hdKeypath, path)) { + strErr = "Error reading wallet database: keymeta with invalid HD keypath"; + return false; + } + } + + // Extract the index and internal from the path + // Path string is m/0'/k'/i' + // Path vector is [0', k', i'] (but as ints OR'd with the hardened bit + // k == 0 for external, 1 for internal. i is the index + if (path.size() != 3) { + strErr = "Error reading wallet database: keymeta found with unexpected path"; + return false; + } + if (path[0] != 0x80000000) { + strErr = strprintf("Unexpected path index of 0x%08x (expected 0x80000000) for the element at index 0", path[0]); + return false; + } + if (path[1] != 0x80000000 && path[1] != (1 | 0x80000000)) { + strErr = strprintf("Unexpected path index of 0x%08x (expected 0x80000000 or 0x80000001) for the element at index 1", path[1]); + return false; + } + if ((path[2] & 0x80000000) == 0) { + strErr = strprintf("Unexpected path index of 0x%08x (expected to be greater than or equal to 0x80000000)", path[2]); + return false; + } + internal = path[1] == (1 | 0x80000000); + index = path[2] & ~0x80000000; + } + + // Insert a new CHDChain, or get the one that already exists + auto ins = wss.m_hd_chains.emplace(keyMeta.hd_seed_id, CHDChain()); + CHDChain& chain = ins.first->second; + if (ins.second) { + // For new chains, we want to default to VERSION_HD_BASE until we see an internal + chain.nVersion = CHDChain::VERSION_HD_BASE; + chain.seed_id = keyMeta.hd_seed_id; + } + if (internal) { + chain.nVersion = CHDChain::VERSION_HD_CHAIN_SPLIT; + chain.nInternalChainCounter = std::max(chain.nInternalChainCounter, index); + } else { + chain.nExternalChainCounter = std::max(chain.nExternalChainCounter, index); + } + } } else if (strType == DBKeys::WATCHMETA) { CScript script; ssKey >> script; @@ -581,6 +672,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, return true; } +bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr) +{ + CWalletScanState dummy_wss; + LOCK(pwallet->cs_wallet); + return ReadKeyValue(pwallet, ssKey, ssValue, dummy_wss, strType, strErr); +} + bool WalletBatch::IsKeyType(const std::string& strType) { return (strType == DBKeys::KEY || @@ -728,10 +826,24 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) result = DBErrors::CORRUPT; } + // Set the inactive chain + if (wss.m_hd_chains.size() > 0) { + LegacyScriptPubKeyMan* legacy_spkm = pwallet->GetLegacyScriptPubKeyMan(); + if (!legacy_spkm) { + pwallet->WalletLogPrintf("Inactive HD Chains found but no Legacy ScriptPubKeyMan\n"); + return DBErrors::CORRUPT; + } + for (const auto& chain_pair : wss.m_hd_chains) { + if (chain_pair.first != pwallet->GetLegacyScriptPubKeyMan()->GetHDChain().seed_id) { + pwallet->GetLegacyScriptPubKeyMan()->AddInactiveHDChain(chain_pair.second); + } + } + } + return result; } -DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx) +DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWalletTx>& vWtx) { DBErrors result = DBErrors::LOAD_OK; @@ -769,12 +881,9 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CW if (strType == DBKeys::TX) { uint256 hash; ssKey >> hash; - - CWalletTx wtx(nullptr /* pwallet */, MakeTransactionRef()); - ssValue >> wtx; - vTxHash.push_back(hash); - vWtx.push_back(wtx); + vWtx.emplace_back(nullptr /* wallet */, nullptr /* tx */); + ssValue >> vWtx.back(); } } pcursor->close(); @@ -793,7 +902,7 @@ DBErrors WalletBatch::ZapSelectTx(std::vector<uint256>& vTxHashIn, std::vector<u { // build list of wallet TXs and hashes std::vector<uint256> vTxHash; - std::vector<CWalletTx> vWtx; + std::list<CWalletTx> vWtx; DBErrors err = FindWalletTx(vTxHash, vWtx); if (err != DBErrors::LOAD_OK) { return err; @@ -827,7 +936,7 @@ DBErrors WalletBatch::ZapSelectTx(std::vector<uint256>& vTxHashIn, std::vector<u return DBErrors::LOAD_OK; } -DBErrors WalletBatch::ZapWalletTx(std::vector<CWalletTx>& vWtx) +DBErrors WalletBatch::ZapWalletTx(std::list<CWalletTx>& vWtx) { // build list of wallet TXs std::vector<uint256> vTxHash; @@ -874,53 +983,14 @@ void MaybeCompactWalletDB() fOneThread = false; } -// -// Try to (very carefully!) recover wallet file if there is a problem. -// -bool WalletBatch::Recover(const fs::path& wallet_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename) -{ - return BerkeleyBatch::Recover(wallet_path, callbackDataIn, recoverKVcallback, out_backup_filename); -} - -bool WalletBatch::Recover(const fs::path& wallet_path, std::string& out_backup_filename) -{ - // recover without a key filter callback - // results in recovering all record types - return WalletBatch::Recover(wallet_path, nullptr, nullptr, out_backup_filename); -} - -bool WalletBatch::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue) -{ - CWallet *dummyWallet = reinterpret_cast<CWallet*>(callbackData); - CWalletScanState dummyWss; - std::string strType, strErr; - bool fReadOK; - { - // Required in LoadKeyMetadata(): - LOCK(dummyWallet->cs_wallet); - fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue, - dummyWss, strType, strErr); - } - if (!IsKeyType(strType) && strType != DBKeys::HDCHAIN) { - return false; - } - if (!fReadOK) - { - LogPrintf("WARNING: WalletBatch::Recover skipping %s: %s\n", strType, strErr); - return false; - } - - return true; -} - -bool WalletBatch::VerifyEnvironment(const fs::path& wallet_path, std::string& errorStr) +bool WalletBatch::VerifyEnvironment(const fs::path& wallet_path, bilingual_str& errorStr) { return BerkeleyBatch::VerifyEnvironment(wallet_path, errorStr); } -bool WalletBatch::VerifyDatabaseFile(const fs::path& wallet_path, std::vector<std::string>& warnings, std::string& errorStr) +bool WalletBatch::VerifyDatabaseFile(const fs::path& wallet_path, bilingual_str& errorStr) { - return BerkeleyBatch::VerifyDatabaseFile(wallet_path, warnings, errorStr, WalletBatch::Recover); + return BerkeleyBatch::VerifyDatabaseFile(wallet_path, errorStr); } bool WalletBatch::WriteDestData(const std::string &address, const std::string &key, const std::string &value) diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 2701481c58..b95ed24d12 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -98,15 +98,13 @@ public: int nVersion; CHDChain() { SetNull(); } - ADD_SERIALIZE_METHODS; - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) + + SERIALIZE_METHODS(CHDChain, obj) { - READWRITE(this->nVersion); - READWRITE(nExternalChainCounter); - READWRITE(seed_id); - if (this->nVersion >= VERSION_HD_CHAIN_SPLIT) - READWRITE(nInternalChainCounter); + READWRITE(obj.nVersion, obj.nExternalChainCounter, obj.seed_id); + if (obj.nVersion >= VERSION_HD_CHAIN_SPLIT) { + READWRITE(obj.nInternalChainCounter); + } } void SetNull() @@ -116,6 +114,11 @@ public: nInternalChainCounter = 0; seed_id.SetNull(); } + + bool operator==(const CHDChain& chain) const + { + return seed_id == chain.seed_id; + } }; class CKeyMetadata @@ -142,21 +145,16 @@ public: nCreateTime = nCreateTime_; } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(this->nVersion); - READWRITE(nCreateTime); - if (this->nVersion >= VERSION_WITH_HDDATA) - { - READWRITE(hdKeypath); - READWRITE(hd_seed_id); + SERIALIZE_METHODS(CKeyMetadata, obj) + { + READWRITE(obj.nVersion, obj.nCreateTime); + if (obj.nVersion >= VERSION_WITH_HDDATA) { + READWRITE(obj.hdKeypath, obj.hd_seed_id); } - if (this->nVersion >= VERSION_WITH_KEY_ORIGIN) + if (obj.nVersion >= VERSION_WITH_KEY_ORIGIN) { - READWRITE(key_origin); - READWRITE(has_key_origin); + READWRITE(obj.key_origin); + READWRITE(obj.has_key_origin); } } @@ -260,21 +258,15 @@ public: bool WriteActiveScriptPubKeyMan(uint8_t type, const uint256& id, bool internal); DBErrors LoadWallet(CWallet* pwallet); - DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx); - DBErrors ZapWalletTx(std::vector<CWalletTx>& vWtx); + DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWalletTx>& vWtx); + DBErrors ZapWalletTx(std::list<CWalletTx>& vWtx); DBErrors ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut); - /* Try to (very carefully!) recover wallet database (with a possible key type filter) */ - static bool Recover(const fs::path& wallet_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename); - /* Recover convenience-function to bypass the key filter callback, called when verify fails, recovers everything */ - static bool Recover(const fs::path& wallet_path, std::string& out_backup_filename); - /* Recover filter (used as callback), will only let keys (cryptographical keys) as KV/key-type pass through */ - static bool RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue); /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ static bool IsKeyType(const std::string& strType); /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path& wallet_path, std::string& errorStr); + static bool VerifyEnvironment(const fs::path& wallet_path, bilingual_str& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& wallet_path, std::vector<std::string>& warnings, std::string& errorStr); + static bool VerifyDatabaseFile(const fs::path& wallet_path, bilingual_str& errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); @@ -294,4 +286,7 @@ private: //! Compacts BDB state so that wallet.dat is self-contained (if there are changes) void MaybeCompactWalletDB(); +//! Unserialize a given Key-Value pair and load it into the wallet +bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr); + #endif // BITCOIN_WALLET_WALLETDB_H diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 65643669c2..be07c28503 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -4,6 +4,8 @@ #include <fs.h> #include <util/system.h> +#include <util/translation.h> +#include <wallet/salvage.h> #include <wallet/wallet.h> #include <wallet/walletutil.h> @@ -102,6 +104,27 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->m_address_book.size()); } +static bool SalvageWallet(const fs::path& path) +{ + // Create a Database handle to allow for the db to be initialized before recovery + std::unique_ptr<WalletDatabase> database = WalletDatabase::Create(path); + + // Initialize the environment before recovery + bilingual_str error_string; + try { + WalletBatch::VerifyEnvironment(path, error_string); + } catch (const fs::filesystem_error& e) { + error_string = Untranslated(strprintf("Error loading wallet. %s", fsbridge::get_filesystem_error_message(e))); + } + if (!error_string.original.empty()) { + tfm::format(std::cerr, "Failed to open wallet for salvage :%s\n", error_string.original); + return false; + } + + // Perform the recovery + return RecoverDatabaseFile(path); +} + bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) { fs::path path = fs::absolute(name, GetWalletDir()); @@ -112,20 +135,25 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) WalletShowInfo(wallet_instance.get()); wallet_instance->Flush(true); } - } else if (command == "info") { + } else if (command == "info" || command == "salvage") { if (!fs::exists(path)) { tfm::format(std::cerr, "Error: no wallet file at %s\n", name); return false; } - std::string error; + bilingual_str error; if (!WalletBatch::VerifyEnvironment(path, error)) { - tfm::format(std::cerr, "Error loading %s. Is wallet being used by other process?\n", name); + tfm::format(std::cerr, "%s\nError loading %s. Is wallet being used by other process?\n", error.original, name); return false; } - std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); - if (!wallet_instance) return false; - WalletShowInfo(wallet_instance.get()); - wallet_instance->Flush(true); + + if (command == "info") { + std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); + if (!wallet_instance) return false; + WalletShowInfo(wallet_instance.get()); + wallet_instance->Flush(true); + } else if (command == "salvage") { + return SalvageWallet(path); + } } else { tfm::format(std::cerr, "Invalid command: %s\n", command); return false; diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index d7e07ed04c..a4e4fda8a1 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -92,32 +92,28 @@ class WalletDescriptor { public: std::shared_ptr<Descriptor> descriptor; - uint64_t creation_time; - int32_t range_start; // First item in range; start of range, inclusive, i.e. [range_start, range_end). This never changes. - int32_t range_end; // Item after the last; end of range, exclusive, i.e. [range_start, range_end). This will increment with each TopUp() - int32_t next_index; // Position of the next item to generate + uint64_t creation_time = 0; + int32_t range_start = 0; // First item in range; start of range, inclusive, i.e. [range_start, range_end). This never changes. + int32_t range_end = 0; // Item after the last; end of range, exclusive, i.e. [range_start, range_end). This will increment with each TopUp() + int32_t next_index = 0; // Position of the next item to generate DescriptorCache cache; - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - if (ser_action.ForRead()) { - std::string desc; - std::string error; - READWRITE(desc); - FlatSigningProvider keys; - descriptor = Parse(desc, keys, error, true); - if (!descriptor) { - throw std::ios_base::failure("Invalid descriptor: " + error); - } - } else { - READWRITE(descriptor->ToString()); + void DeserializeDescriptor(const std::string& str) + { + std::string error; + FlatSigningProvider keys; + descriptor = Parse(str, keys, error, true); + if (!descriptor) { + throw std::ios_base::failure("Invalid descriptor: " + error); } - READWRITE(creation_time); - READWRITE(next_index); - READWRITE(range_start); - READWRITE(range_end); + } + + SERIALIZE_METHODS(WalletDescriptor, obj) + { + std::string descriptor_str; + SER_WRITE(obj, descriptor_str = obj.descriptor->ToString()); + READWRITE(descriptor_str, obj.creation_time, obj.next_index, obj.range_start, obj.range_end); + SER_READ(obj, obj.DeserializeDescriptor(descriptor_str)); } WalletDescriptor() {} diff --git a/test/README.md b/test/README.md index e1dab92a06..0210907878 100644 --- a/test/README.md +++ b/test/README.md @@ -225,6 +225,10 @@ gdb /home/example/bitcoind <pid> Note: gdb attach step may require ptrace_scope to be modified, or `sudo` preceding the `gdb`. See this link for considerations: https://www.kernel.org/doc/Documentation/security/Yama.txt +Often while debugging rpc calls from functional tests, the test might reach timeout before +process can return a response. Use `--timeout-factor 0` to disable all rpc timeouts for that partcular +functional test. Ex: `test/functional/wallet_hd.py --timeout-factor 0`. + ##### Profiling An easy way to profile node performance during functional tests is provided diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 70dfe81d4e..5d782026dc 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -15,7 +15,7 @@ from collections import defaultdict # Avoid wildcard * imports from test_framework.blocktools import (create_block, create_coinbase) -from test_framework.messages import CInv +from test_framework.messages import CInv, MSG_BLOCK from test_framework.mininode import ( P2PInterface, mininode_lock, @@ -198,7 +198,7 @@ class ExampleTest(BitcoinTestFramework): getdata_request = msg_getdata() for block in blocks: - getdata_request.inv.append(CInv(2, block)) + getdata_request.inv.append(CInv(MSG_BLOCK, block)) self.nodes[2].p2p.send_message(getdata_request) # wait_until() will loop until a predicate condition is met. Use it to test properties of the diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index 166c28d376..596ff206f2 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -6,7 +6,11 @@ Test various backwards compatibility scenarios. Download the previous node binaries: -contrib/devtools/previous_release.sh -b v0.19.0.1 v0.18.1 v0.17.1 +contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 + +v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py. +Due to a hardfork in regtest, it can't be used to sync nodes. + Due to RPC changes introduced in various versions the below tests won't work for older versions without some patches or workarounds. @@ -18,60 +22,46 @@ needs an older patch version. import os import shutil -from test_framework.test_framework import BitcoinTestFramework, SkipTest +from test_framework.test_framework import BitcoinTestFramework from test_framework.descriptors import descsum_create from test_framework.util import ( + adjust_bitcoin_conf_for_pre_17, assert_equal, sync_blocks, - sync_mempools + sync_mempools, ) + class BackwardsCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 5 + self.num_nodes = 6 # Add new version after each release: self.extra_args = [ ["-addresstype=bech32"], # Pre-release: use to mine blocks ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # Pre-release: use to receive coins, swap wallets, etc - ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.19.0.1 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.19.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.18.1 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32"] # v0.17.1 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.17.1 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.16.3 ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_previous_releases() def setup_nodes(self): - if os.getenv("TEST_PREVIOUS_RELEASES") == "false": - raise SkipTest("backwards compatibility tests") - - releases_path = os.getenv("PREVIOUS_RELEASES_DIR") or os.getcwd() + "/releases" - if not os.path.isdir(releases_path): - if os.getenv("TEST_PREVIOUS_RELEASES") == "true": - raise AssertionError("TEST_PREVIOUS_RELEASES=1 but releases missing: " + releases_path) - raise SkipTest("This test requires binaries for previous releases") - self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ None, None, - 190000, + 190100, 180100, - 170100 - ], binary=[ - self.options.bitcoind, - self.options.bitcoind, - releases_path + "/v0.19.0.1/bin/bitcoind", - releases_path + "/v0.18.1/bin/bitcoind", - releases_path + "/v0.17.1/bin/bitcoind" - ], binary_cli=[ - self.options.bitcoincli, - self.options.bitcoincli, - releases_path + "/v0.19.0.1/bin/bitcoin-cli", - releases_path + "/v0.18.1/bin/bitcoin-cli", - releases_path + "/v0.17.1/bin/bitcoin-cli" + 170100, + 160300, ]) + # adapt bitcoin.conf, because older bitcoind's don't recognize config sections + adjust_bitcoin_conf_for_pre_17(self.nodes[5].bitcoinconf) self.start_nodes() @@ -84,10 +74,11 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): res = self.nodes[self.num_nodes - 1].getblockchaininfo() assert_equal(res['blocks'], 101) - node_master = self.nodes[self.num_nodes - 4] - node_v19 = self.nodes[self.num_nodes - 3] - node_v18 = self.nodes[self.num_nodes - 2] - node_v17 = self.nodes[self.num_nodes - 1] + node_master = self.nodes[self.num_nodes - 5] + node_v19 = self.nodes[self.num_nodes - 4] + node_v18 = self.nodes[self.num_nodes - 3] + node_v17 = self.nodes[self.num_nodes - 2] + node_v16 = self.nodes[self.num_nodes - 1] self.log.info("Test wallet backwards compatibility...") # Create a number of wallets and open them in older versions: @@ -186,6 +177,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_v19_wallets_dir = os.path.join(node_v19.datadir, "regtest/wallets") node_v18_wallets_dir = os.path.join(node_v18.datadir, "regtest/wallets") node_v17_wallets_dir = os.path.join(node_v17.datadir, "regtest/wallets") + node_v16_wallets_dir = os.path.join(node_v16.datadir, "regtest") node_master.unloadwallet("w1") node_master.unloadwallet("w2") node_v19.unloadwallet("w1_v19") @@ -193,6 +185,13 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_v18.unloadwallet("w1_v18") node_v18.unloadwallet("w2_v18") + # Copy wallets to v0.16 + for wallet in os.listdir(node_master_wallets_dir): + shutil.copytree( + os.path.join(node_master_wallets_dir, wallet), + os.path.join(node_v16_wallets_dir, wallet) + ) + # Copy wallets to v0.17 for wallet in os.listdir(node_master_wallets_dir): shutil.copytree( @@ -311,19 +310,26 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # assert_raises_rpc_error(-4, "Wallet loading failed.", node_v17.loadwallet, 'w3_v18') # Instead, we stop node and try to launch it with the wallet: - self.stop_node(self.num_nodes - 1) + self.stop_node(4) node_v17.assert_start_raises_init_error(["-wallet=w3_v18"], "Error: Error loading w3_v18: Wallet requires newer version of Bitcoin Core") node_v17.assert_start_raises_init_error(["-wallet=w3"], "Error: Error loading w3: Wallet requires newer version of Bitcoin Core") - self.start_node(self.num_nodes - 1) + self.start_node(4) + + # Open most recent wallet in v0.16 (no loadwallet RPC) + self.stop_node(5) + self.start_node(5, extra_args=["-wallet=w2"]) + wallet = node_v16.get_wallet_rpc("w2") + info = wallet.getwalletinfo() + assert info['keypoolsize'] == 1 self.log.info("Test wallet upgrade path...") # u1: regular wallet, created with v0.17 node_v17.rpc.createwallet(wallet_name="u1_v17") wallet = node_v17.get_wallet_rpc("u1_v17") address = wallet.getnewaddress("bech32") - info = wallet.getaddressinfo(address) - hdkeypath = info["hdkeypath"] - pubkey = info["pubkey"] + v17_info = wallet.getaddressinfo(address) + hdkeypath = v17_info["hdkeypath"] + pubkey = v17_info["pubkey"] # Copy the 0.17 wallet to the last Bitcoin Core version and open it: node_v17.unloadwallet("u1_v17") @@ -337,6 +343,18 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")" assert_equal(info["desc"], descsum_create(descriptor)) + # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it + node_master.unloadwallet("u1_v17") + shutil.rmtree(os.path.join(node_v17_wallets_dir, "u1_v17")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "u1_v17"), + os.path.join(node_v17_wallets_dir, "u1_v17") + ) + node_v17.loadwallet("u1_v17") + wallet = node_v17.get_wallet_rpc("u1_v17") + info = wallet.getaddressinfo(address) + assert_equal(info, v17_info) + # Copy the 0.19 wallet to the last Bitcoin Core version and open it: shutil.copytree( os.path.join(node_v19_wallets_dir, "w1_v19"), @@ -346,5 +364,16 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): wallet = node_master.get_wallet_rpc("w1_v19") assert wallet.getaddressinfo(address_18075)["solvable"] + # Now copy that same wallet back to 0.19 to make sure no automatic upgrade breaks it + node_master.unloadwallet("w1_v19") + shutil.rmtree(os.path.join(node_v19_wallets_dir, "w1_v19")) + shutil.copytree( + os.path.join(node_master_wallets_dir, "w1_v19"), + os.path.join(node_v19_wallets_dir, "w1_v19") + ) + node_v19.loadwallet("w1_v19") + wallet = node_v19.get_wallet_rpc("w1_v19") + assert wallet.getaddressinfo(address_18075)["solvable"] + if __name__ == '__main__': BackwardsCompatibilityTest().main() diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index 1138b0f0ea..82f1331685 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -16,10 +16,8 @@ import sys import tempfile import urllib -from test_framework.test_framework import ( - BitcoinTestFramework, -) -from test_framework.util import assert_equal, wait_until +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal class LoadblockTest(BitcoinTestFramework): @@ -75,7 +73,7 @@ class LoadblockTest(BitcoinTestFramework): self.log.info("Restart second, unsynced node with bootstrap file") self.stop_node(1) self.start_node(1, ["-loadblock=" + bootstrap_file]) - wait_until(lambda: self.nodes[1].getblockcount() == 100) + assert_equal(self.nodes[1].getblockcount(), 100) # start_node is blocking on all block files being imported assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 100) assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index d4a8f8a715..9579a1715d 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -13,7 +13,7 @@ if uploadtarget has been reached. from collections import defaultdict import time -from test_framework.messages import CInv, msg_getdata +from test_framework.messages import CInv, MSG_BLOCK, msg_getdata from test_framework.mininode import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, mine_large_block @@ -84,7 +84,7 @@ class MaxUploadTest(BitcoinTestFramework): # the same big old block too many times (expect: disconnect) getdata_request = msg_getdata() - getdata_request.inv.append(CInv(2, big_old_block)) + getdata_request.inv.append(CInv(MSG_BLOCK, big_old_block)) max_bytes_per_day = 800*1024*1024 daily_buffer = 144 * 4000000 @@ -109,7 +109,7 @@ class MaxUploadTest(BitcoinTestFramework): # Requesting the current block on p2p_conns[1] should succeed indefinitely, # even when over the max upload target. # We'll try 800 times - getdata_request.inv = [CInv(2, big_new_block)] + getdata_request.inv = [CInv(MSG_BLOCK, big_new_block)] for i in range(800): p2p_conns[1].send_and_ping(getdata_request) assert_equal(p2p_conns[1].block_receive_map[big_new_block], i+1) @@ -117,7 +117,7 @@ class MaxUploadTest(BitcoinTestFramework): self.log.info("Peer 1 able to repeatedly download new block") # But if p2p_conns[1] tries for an old block, it gets disconnected too. - getdata_request.inv = [CInv(2, big_old_block)] + getdata_request.inv = [CInv(MSG_BLOCK, big_old_block)] p2p_conns[1].send_message(getdata_request) p2p_conns[1].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 1) @@ -145,12 +145,12 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].add_p2p_connection(TestP2PConn()) #retrieve 20 blocks which should be enough to break the 1MB limit - getdata_request.inv = [CInv(2, big_new_block)] + getdata_request.inv = [CInv(MSG_BLOCK, big_new_block)] for i in range(20): self.nodes[0].p2p.send_and_ping(getdata_request) assert_equal(self.nodes[0].p2p.block_receive_map[big_new_block], i+1) - getdata_request.inv = [CInv(2, big_old_block)] + getdata_request.inv = [CInv(MSG_BLOCK, big_old_block)] self.nodes[0].p2p.send_and_ping(getdata_request) assert_equal(len(self.nodes[0].getpeerinfo()), 1) #node is still connected because of the whitelist diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index b110a559c0..47200b6cc6 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -5,12 +5,14 @@ """Test the -alertnotify, -blocknotify and -walletnotify options.""" import os -from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE, keyhash_to_p2pkh from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, wait_until, connect_nodes, + disconnect_nodes, + hex_str_to_bytes, ) # Linux allow all characters other than \x00 @@ -81,8 +83,72 @@ class NotificationsTest(BitcoinTestFramework): # directory content should equal the generated transaction hashes txids_rpc = list(map(lambda t: notify_outputname(self.wallet, t['txid']), self.nodes[1].listtransactions("*", block_count))) assert_equal(sorted(txids_rpc), sorted(os.listdir(self.walletnotify_dir))) + for tx_file in os.listdir(self.walletnotify_dir): + os.remove(os.path.join(self.walletnotify_dir, tx_file)) + + # Conflicting transactions tests. Give node 0 same wallet seed as + # node 1, generate spends from node 0, and check notifications + # triggered by node 1 + self.log.info("test -walletnotify with conflicting transactions") + self.nodes[0].sethdseed(seed=self.nodes[1].dumpprivkey(keyhash_to_p2pkh(hex_str_to_bytes(self.nodes[1].getwalletinfo()['hdseedid'])[::-1]))) + self.nodes[0].rescanblockchain() + self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_UNSPENDABLE) + + # Generate transaction on node 0, sync mempools, and check for + # notification on node 1. + tx1 = self.nodes[0].sendtoaddress(address=ADDRESS_BCRT1_UNSPENDABLE, amount=1, replaceable=True) + assert_equal(tx1 in self.nodes[0].getrawmempool(), True) + self.sync_mempools() + self.expect_wallet_notify([tx1]) + + # Generate bump transaction, sync mempools, and check for bump1 + # notification. In the future, per + # https://github.com/bitcoin/bitcoin/pull/9371, it might be better + # to have notifications for both tx1 and bump1. + bump1 = self.nodes[0].bumpfee(tx1)["txid"] + assert_equal(bump1 in self.nodes[0].getrawmempool(), True) + self.sync_mempools() + self.expect_wallet_notify([bump1]) + + # Add bump1 transaction to new block, checking for a notification + # and the correct number of confirmations. + self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + self.sync_blocks() + self.expect_wallet_notify([bump1]) + assert_equal(self.nodes[1].gettransaction(bump1)["confirmations"], 1) + + # Generate a second transaction to be bumped. + tx2 = self.nodes[0].sendtoaddress(address=ADDRESS_BCRT1_UNSPENDABLE, amount=1, replaceable=True) + assert_equal(tx2 in self.nodes[0].getrawmempool(), True) + self.sync_mempools() + self.expect_wallet_notify([tx2]) + + # Bump tx2 as bump2 and generate a block on node 0 while + # disconnected, then reconnect and check for notifications on node 1 + # about newly confirmed bump2 and newly conflicted tx2. Currently + # only the bump2 notification is sent. Ideally, notifications would + # be sent both for bump2 and tx2, which was the previous behavior + # before being broken by an accidental change in PR + # https://github.com/bitcoin/bitcoin/pull/16624. The bug is reported + # in issue https://github.com/bitcoin/bitcoin/issues/18325. + disconnect_nodes(self.nodes[0], 1) + bump2 = self.nodes[0].bumpfee(tx2)["txid"] + self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + assert_equal(self.nodes[0].gettransaction(bump2)["confirmations"], 1) + assert_equal(tx2 in self.nodes[1].getrawmempool(), True) + connect_nodes(self.nodes[0], 1) + self.sync_blocks() + self.expect_wallet_notify([bump2]) + assert_equal(self.nodes[1].gettransaction(bump2)["confirmations"], 1) # TODO: add test for `-alertnotify` large fork notifications + def expect_wallet_notify(self, tx_ids): + wait_until(lambda: len(os.listdir(self.walletnotify_dir)) >= len(tx_ids), timeout=10) + assert_equal(sorted(notify_outputname(self.wallet, tx_id) for tx_id in tx_ids), sorted(os.listdir(self.walletnotify_dir))) + for tx_file in os.listdir(self.walletnotify_dir): + os.remove(os.path.join(self.walletnotify_dir, tx_file)) + + if __name__ == '__main__': NotificationsTest().main() diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index 940b403f9c..31cea8d1b7 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -10,10 +10,10 @@ """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import wait_until +from test_framework.util import assert_equal -class ReindexTest(BitcoinTestFramework): +class ReindexTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 @@ -24,7 +24,7 @@ class ReindexTest(BitcoinTestFramework): self.stop_nodes() extra_args = [["-reindex-chainstate" if justchainstate else "-reindex"]] self.start_nodes(extra_args) - wait_until(lambda: self.nodes[0].getblockcount() == blockcount) + assert_equal(self.nodes[0].getblockcount(), blockcount) # start_node is blocking on reindex self.log.info("Success") def run_test(self): diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index fdd86310c0..24c357091f 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -108,12 +108,7 @@ class SegWitTest(BitcoinTestFramework): assert tmpl['sigoplimit'] == 20000 assert tmpl['transactions'][0]['hash'] == txid assert tmpl['transactions'][0]['sigops'] == 2 - tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert tmpl['sizelimit'] == 1000000 - assert 'weightlimit' not in tmpl - assert tmpl['sigoplimit'] == 20000 - assert tmpl['transactions'][0]['hash'] == txid - assert tmpl['transactions'][0]['sigops'] == 2 + assert '!segwit' not in tmpl['rules'] self.nodes[0].generate(1) # block 162 balance_presetup = self.nodes[0].getbalance() @@ -213,6 +208,7 @@ class SegWitTest(BitcoinTestFramework): assert tmpl['sigoplimit'] == 80000 assert tmpl['transactions'][0]['txid'] == txid assert tmpl['transactions'][0]['sigops'] == 8 + assert '!segwit' in tmpl['rules'] self.nodes[0].generate(1) # Mine a block to clear the gbt cache diff --git a/test/functional/framework_test_script.py b/test/functional/framework_test_script.py deleted file mode 100755 index 9d916c0022..0000000000 --- a/test/functional/framework_test_script.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Tests for test_framework.script.""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.script import bn2vch -from test_framework.util import assert_equal - -def test_bn2vch(): - assert_equal(bn2vch(0), bytes([])) - assert_equal(bn2vch(1), bytes([0x01])) - assert_equal(bn2vch(-1), bytes([0x81])) - assert_equal(bn2vch(0x7F), bytes([0x7F])) - assert_equal(bn2vch(-0x7F), bytes([0xFF])) - assert_equal(bn2vch(0x80), bytes([0x80, 0x00])) - assert_equal(bn2vch(-0x80), bytes([0x80, 0x80])) - assert_equal(bn2vch(0xFF), bytes([0xFF, 0x00])) - assert_equal(bn2vch(-0xFF), bytes([0xFF, 0x80])) - assert_equal(bn2vch(0x100), bytes([0x00, 0x01])) - assert_equal(bn2vch(-0x100), bytes([0x00, 0x81])) - assert_equal(bn2vch(0x7FFF), bytes([0xFF, 0x7F])) - assert_equal(bn2vch(-0x8000), bytes([0x00, 0x80, 0x80])) - assert_equal(bn2vch(-0x7FFFFF), bytes([0xFF, 0xFF, 0xFF])) - assert_equal(bn2vch(0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x00])) - assert_equal(bn2vch(-0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x80])) - assert_equal(bn2vch(0xFFFFFFFF), bytes([0xFF, 0xFF, 0xFF, 0xFF, 0x00])) - - assert_equal(bn2vch(123456789), bytes([0x15, 0xCD, 0x5B, 0x07])) - assert_equal(bn2vch(-54321), bytes([0x31, 0xD4, 0x80])) - -class FrameworkTestScript(BitcoinTestFramework): - def setup_network(self): - pass - - def set_test_params(self): - self.num_nodes = 0 - - def run_test(self): - test_bn2vch() - -if __name__ == '__main__': - FrameworkTestScript().main() diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 1c94305220..7530e7daf6 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -67,6 +67,7 @@ class TestBitcoinCli(BitcoinTestFramework): if self.is_wallet_compiled(): self.log.info("Test -getinfo and bitcoin-cli getwalletinfo return expected wallet info") assert_equal(cli_get_info['balance'], BALANCE) + assert 'balances' not in cli_get_info.keys() wallet_info = self.nodes[0].getwalletinfo() assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) assert_equal(cli_get_info['unlocked_until'], wallet_info['unlocked_until']) @@ -76,42 +77,60 @@ class TestBitcoinCli(BitcoinTestFramework): # Setup to test -getinfo and -rpcwallet= with multiple wallets. wallets = ['', 'Encrypted', 'secret'] - amounts = [Decimal('59.999928'), Decimal(9), Decimal(31)] + amounts = [BALANCE + Decimal('9.999928'), Decimal(9), Decimal(31)] self.nodes[0].createwallet(wallet_name=wallets[1]) self.nodes[0].createwallet(wallet_name=wallets[2]) w1 = self.nodes[0].get_wallet_rpc(wallets[0]) w2 = self.nodes[0].get_wallet_rpc(wallets[1]) w3 = self.nodes[0].get_wallet_rpc(wallets[2]) w1.walletpassphrase(password, self.rpc_timeout) + w2.encryptwallet(password) w1.sendtoaddress(w2.getnewaddress(), amounts[1]) w1.sendtoaddress(w3.getnewaddress(), amounts[2]) # Mine a block to confirm; adds a block reward (50 BTC) to the default wallet. self.nodes[0].generate(1) - self.log.info("Test -getinfo with multiple wallets loaded returns no balance") - assert_equal(set(self.nodes[0].listwallets()), set(wallets)) - assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli().keys() - self.log.info("Test -getinfo with multiple wallets and -rpcwallet returns specified wallet balance") for i in range(len(wallets)): - cli_get_info = self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[i])) + cli_get_info = self.nodes[0].cli('-getinfo', '-rpcwallet={}'.format(wallets[i])).send_cli() + assert 'balances' not in cli_get_info.keys() assert_equal(cli_get_info['balance'], amounts[i]) - self.log.info("Test -getinfo with multiple wallets and -rpcwallet=non-existing-wallet returns no balance") - assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli('-rpcwallet=does-not-exist').keys() + self.log.info("Test -getinfo with multiple wallets and -rpcwallet=non-existing-wallet returns no balances") + cli_get_info_keys = self.nodes[0].cli('-getinfo', '-rpcwallet=does-not-exist').send_cli().keys() + assert 'balance' not in cli_get_info_keys + assert 'balances' not in cli_get_info_keys - self.log.info("Test -getinfo after unloading all wallets except a non-default one returns its balance") + self.log.info("Test -getinfo with multiple wallets returns all loaded wallet names and balances") + assert_equal(set(self.nodes[0].listwallets()), set(wallets)) + cli_get_info = self.nodes[0].cli('-getinfo').send_cli() + assert 'balance' not in cli_get_info.keys() + assert_equal(cli_get_info['balances'], {k: v for k, v in zip(wallets, amounts)}) + + # Unload the default wallet and re-verify. self.nodes[0].unloadwallet(wallets[0]) + assert wallets[0] not in self.nodes[0].listwallets() + cli_get_info = self.nodes[0].cli('-getinfo').send_cli() + assert 'balance' not in cli_get_info.keys() + assert_equal(cli_get_info['balances'], {k: v for k, v in zip(wallets[1:], amounts[1:])}) + + self.log.info("Test -getinfo after unloading all wallets except a non-default one returns its balance") self.nodes[0].unloadwallet(wallets[2]) assert_equal(self.nodes[0].listwallets(), [wallets[1]]) - assert_equal(self.nodes[0].cli('-getinfo').send_cli()['balance'], amounts[1]) - - self.log.info("Test -getinfo -rpcwallet=remaining-non-default-wallet returns its balance") - assert_equal(self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[1]))['balance'], amounts[1]) - - self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balance") - assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[2])).keys() + cli_get_info = self.nodes[0].cli('-getinfo').send_cli() + assert 'balances' not in cli_get_info.keys() + assert_equal(cli_get_info['balance'], amounts[1]) + + self.log.info("Test -getinfo with -rpcwallet=remaining-non-default-wallet returns only its balance") + cli_get_info = self.nodes[0].cli('-getinfo', '-rpcwallet={}'.format(wallets[1])).send_cli() + assert 'balances' not in cli_get_info.keys() + assert_equal(cli_get_info['balance'], amounts[1]) + + self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balances") + cli_get_info = self.nodes[0].cli('-getinfo', '-rpcwallet={}'.format(wallets[2])).send_cli() + assert 'balance' not in cli_get_info_keys + assert 'balances' not in cli_get_info_keys else: self.log.info("*** Wallet not compiled; cli getwalletinfo and -getinfo wallet tests skipped") self.nodes[0].generate(1) # maintain block parity with the wallet_compiled conditional branch diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index a07dad18d6..5b7216b253 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -7,6 +7,7 @@ from decimal import Decimal from test_framework.messages import COIN +from test_framework.mininode import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -58,6 +59,7 @@ class MempoolPackagesTest(BitcoinTestFramework): def run_test(self): # Mine some blocks and have them mature. + self.nodes[0].add_p2p_connection(P2PTxInvStore()) # keep track of invs self.nodes[0].generate(101) utxo = self.nodes[0].listunspent(10) txid = utxo[0]['txid'] @@ -72,6 +74,10 @@ class MempoolPackagesTest(BitcoinTestFramework): value = sent_value chain.append(txid) + # Wait until mempool transactions have passed initial broadcast (sent inv and received getdata) + # Otherwise, getrawmempool may be inconsistent with getmempoolentry if unbroadcast changes in between + self.nodes[0].p2p.wait_for_broadcast(chain) + # Check mempool has MAX_ANCESTORS transactions in it, and descendant and ancestor # count and fees should look correct mempool = self.nodes[0].getrawmempool(True) @@ -212,6 +218,10 @@ class MempoolPackagesTest(BitcoinTestFramework): for tx in chain[:MAX_ANCESTORS_CUSTOM]: assert tx in mempool1 # TODO: more detailed check of node1's mempool (fees etc.) + # check transaction unbroadcast info (should be false if in both mempools) + mempool = self.nodes[0].getrawmempool(True) + for tx in mempool: + assert_equal(mempool[tx]['unbroadcast'], False) # TODO: test ancestor size limits diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 99003d2d1f..3969da2eb0 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -95,8 +95,8 @@ class MempoolPersistTest(BitcoinTestFramework): self.start_node(1, extra_args=["-persistmempool=0"]) self.start_node(0) self.start_node(2) - wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"], timeout=1) - wait_until(lambda: self.nodes[2].getmempoolinfo()["loaded"], timeout=1) + assert self.nodes[0].getmempoolinfo()["loaded"] # start_node is blocking on the mempool being loaded + assert self.nodes[2].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 6) assert_equal(len(self.nodes[2].getrawmempool()), 5) # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now: @@ -117,13 +117,13 @@ class MempoolPersistTest(BitcoinTestFramework): self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() self.start_node(0, extra_args=["-persistmempool=0", "-disablewallet"]) - wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) + assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 0) self.log.debug("Stop-start node0. Verify that it has the transactions in its mempool.") self.stop_nodes() self.start_node(0) - wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) + assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 6) mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') @@ -137,7 +137,7 @@ class MempoolPersistTest(BitcoinTestFramework): os.rename(mempooldat0, mempooldat1) self.stop_nodes() self.start_node(1, extra_args=[]) - wait_until(lambda: self.nodes[1].getmempoolinfo()["loaded"]) + assert self.nodes[1].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[1].getrawmempool()), 6) self.log.debug("Prevent bitcoind from writing mempool.dat to disk. Verify that `savemempool` fails") diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index a561f28b91..dedf5b8a47 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -53,6 +53,13 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): txFS = node.signrawtransactionwithwallet(txF["hex"]) rpc_tx_hsh = node.sendrawtransaction(txFS["hex"]) + # check transactions are in unbroadcast using rpc + mempoolinfo = self.nodes[0].getmempoolinfo() + assert_equal(mempoolinfo['unbroadcastcount'], 2) + mempool = self.nodes[0].getrawmempool(True) + for tx in mempool: + assert_equal(mempool[tx]['unbroadcast'], True) + # check that second node doesn't have these two txns mempool = self.nodes[1].getrawmempool() assert rpc_tx_hsh not in mempool @@ -71,6 +78,11 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): assert rpc_tx_hsh in mempool assert wallet_tx_hsh in mempool + # check that transactions are no longer in first node's unbroadcast set + mempool = self.nodes[0].getrawmempool(True) + for tx in mempool: + assert_equal(mempool[tx]['unbroadcast'], False) + self.log.info("Add another connection & ensure transactions aren't broadcast again") conn = node.add_p2p_connection(P2PTxInvStore()) diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py new file mode 100755 index 0000000000..9ff76b4b3d --- /dev/null +++ b/test/functional/p2p_blockfilters.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Tests NODE_COMPACT_FILTERS (BIP 157/158). + +Tests that a node configured with -blockfilterindex and -peerblockfilters can serve +cfheaders and cfcheckpts. +""" + +from test_framework.messages import ( + FILTER_TYPE_BASIC, + hash256, + msg_getcfcheckpt, + msg_getcfheaders, + ser_uint256, + uint256_from_str, +) +from test_framework.mininode import P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + connect_nodes, + disconnect_nodes, + wait_until, +) + +class CompactFiltersTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.rpc_timeout = 480 + self.num_nodes = 2 + self.extra_args = [ + ["-blockfilterindex", "-peerblockfilters"], + ["-blockfilterindex"], + ] + + def run_test(self): + # Node 0 supports COMPACT_FILTERS, node 1 does not. + node0 = self.nodes[0].add_p2p_connection(P2PInterface()) + node1 = self.nodes[1].add_p2p_connection(P2PInterface()) + + # Nodes 0 & 1 share the same first 999 blocks in the chain. + self.nodes[0].generate(999) + self.sync_blocks(timeout=600) + + # Stale blocks by disconnecting nodes 0 & 1, mining, then reconnecting + disconnect_nodes(self.nodes[0], 1) + + self.nodes[0].generate(1) + wait_until(lambda: self.nodes[0].getblockcount() == 1000) + stale_block_hash = self.nodes[0].getblockhash(1000) + + self.nodes[1].generate(1001) + wait_until(lambda: self.nodes[1].getblockcount() == 2000) + + self.log.info("get cfcheckpt on chain to be re-orged out.") + request = msg_getcfcheckpt( + filter_type=FILTER_TYPE_BASIC, + stop_hash=int(stale_block_hash, 16) + ) + node0.send_and_ping(message=request) + response = node0.last_message['cfcheckpt'] + assert_equal(response.filter_type, request.filter_type) + assert_equal(response.stop_hash, request.stop_hash) + assert_equal(len(response.headers), 1) + + self.log.info("Reorg node 0 to a new chain.") + connect_nodes(self.nodes[0], 1) + self.sync_blocks(timeout=600) + + main_block_hash = self.nodes[0].getblockhash(1000) + assert main_block_hash != stale_block_hash, "node 0 chain did not reorganize" + + self.log.info("Check that peers can fetch cfcheckpt on active chain.") + tip_hash = self.nodes[0].getbestblockhash() + request = msg_getcfcheckpt( + filter_type=FILTER_TYPE_BASIC, + stop_hash=int(tip_hash, 16) + ) + node0.send_and_ping(request) + response = node0.last_message['cfcheckpt'] + assert_equal(response.filter_type, request.filter_type) + assert_equal(response.stop_hash, request.stop_hash) + + main_cfcheckpt = self.nodes[0].getblockfilter(main_block_hash, 'basic')['header'] + tip_cfcheckpt = self.nodes[0].getblockfilter(tip_hash, 'basic')['header'] + assert_equal( + response.headers, + [int(header, 16) for header in (main_cfcheckpt, tip_cfcheckpt)] + ) + + self.log.info("Check that peers can fetch cfcheckpt on stale chain.") + request = msg_getcfcheckpt( + filter_type=FILTER_TYPE_BASIC, + stop_hash=int(stale_block_hash, 16) + ) + node0.send_and_ping(request) + response = node0.last_message['cfcheckpt'] + + stale_cfcheckpt = self.nodes[0].getblockfilter(stale_block_hash, 'basic')['header'] + assert_equal( + response.headers, + [int(header, 16) for header in (stale_cfcheckpt,)] + ) + + self.log.info("Check that peers can fetch cfheaders on active chain.") + request = msg_getcfheaders( + filter_type=FILTER_TYPE_BASIC, + start_height=1, + stop_hash=int(main_block_hash, 16) + ) + node0.send_and_ping(request) + response = node0.last_message['cfheaders'] + assert_equal(len(response.hashes), 1000) + assert_equal( + compute_last_header(response.prev_header, response.hashes), + int(main_cfcheckpt, 16) + ) + + self.log.info("Check that peers can fetch cfheaders on stale chain.") + request = msg_getcfheaders( + filter_type=FILTER_TYPE_BASIC, + start_height=1, + stop_hash=int(stale_block_hash, 16) + ) + node0.send_and_ping(request) + response = node0.last_message['cfheaders'] + assert_equal(len(response.hashes), 1000) + assert_equal( + compute_last_header(response.prev_header, response.hashes), + int(stale_cfcheckpt, 16) + ) + + self.log.info("Requests to node 1 without NODE_COMPACT_FILTERS results in disconnection.") + requests = [ + msg_getcfcheckpt( + filter_type=FILTER_TYPE_BASIC, + stop_hash=int(main_block_hash, 16) + ), + msg_getcfheaders( + filter_type=FILTER_TYPE_BASIC, + start_height=1000, + stop_hash=int(main_block_hash, 16) + ), + ] + for request in requests: + node1 = self.nodes[1].add_p2p_connection(P2PInterface()) + node1.send_message(request) + node1.wait_for_disconnect() + + self.log.info("Check that invalid requests result in disconnection.") + requests = [ + # Requesting too many filter headers results in disconnection. + msg_getcfheaders( + filter_type=FILTER_TYPE_BASIC, + start_height=0, + stop_hash=int(tip_hash, 16) + ), + # Requesting unknown filter type results in disconnection. + msg_getcfcheckpt( + filter_type=255, + stop_hash=int(main_block_hash, 16) + ), + # Requesting unknown hash results in disconnection. + msg_getcfcheckpt( + filter_type=FILTER_TYPE_BASIC, + stop_hash=123456789, + ), + ] + for request in requests: + node0 = self.nodes[0].add_p2p_connection(P2PInterface()) + node0.send_message(request) + node0.wait_for_disconnect() + +def compute_last_header(prev_header, hashes): + """Compute the last filter header from a starting header and a sequence of filter hashes.""" + header = ser_uint256(prev_header) + for filter_hash in hashes: + header = hash256(ser_uint256(filter_hash) + header) + return uint256_from_str(header) + +if __name__ == '__main__': + CompactFiltersTest().main() diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 3258a38e3c..c155dda664 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -57,6 +57,29 @@ class P2PBlocksOnly(BitcoinTestFramework): self.nodes[0].p2p.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) + self.log.info('Check that txs from whitelisted peers are not rejected and relayed to others') + self.log.info("Restarting node 0 with whitelist permission and blocksonly") + self.restart_node(0, ["-persistmempool=0", "-whitelist=127.0.0.1", "-whitelistforcerelay", "-blocksonly"]) + assert_equal(self.nodes[0].getrawmempool(),[]) + first_peer = self.nodes[0].add_p2p_connection(P2PInterface()) + second_peer = self.nodes[0].add_p2p_connection(P2PInterface()) + peer_1_info = self.nodes[0].getpeerinfo()[0] + assert_equal(peer_1_info['whitelisted'], True) + assert_equal(peer_1_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool']) + peer_2_info = self.nodes[0].getpeerinfo()[1] + assert_equal(peer_2_info['whitelisted'], True) + assert_equal(peer_2_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool']) + assert_equal(self.nodes[0].testmempoolaccept([sigtx])[0]['allowed'], True) + txid = self.nodes[0].testmempoolaccept([sigtx])[0]['txid'] + + self.log.info('Check that the tx from whitelisted first_peer is relayed to others (ie.second_peer)') + with self.nodes[0].assert_debug_log(["received getdata"]): + first_peer.send_message(msg_tx(FromHex(CTransaction(), sigtx))) + self.log.info('Check that the whitelisted peer is still connected after sending the transaction') + assert_equal(first_peer.is_connected, True) + second_peer.wait_for_tx(txid) + assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) + self.log.info("Whitelisted peer's transaction is accepted and relayed") if __name__ == '__main__': P2PBlocksOnly().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 66e6f8c424..d77a744758 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -10,7 +10,7 @@ Version 2 compact blocks are post-segwit (wtxids) import random from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment -from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_no_witness_block, msg_no_witness_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_block, msg_blocktxn, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex +from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_no_witness_block, msg_no_witness_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_block, msg_blocktxn, MSG_BLOCK, MSG_CMPCT_BLOCK, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex from test_framework.mininode import mininode_lock, P2PInterface from test_framework.script import CScript, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework @@ -44,7 +44,7 @@ class TestP2PConn(P2PInterface): def on_inv(self, message): for x in self.last_message["inv"].inv: - if x.type == 2: + if x.type == MSG_BLOCK: self.block_announced = True self.announced_blockhashes.add(x.hash) @@ -307,7 +307,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch the compact block using a normal non-announce getdata with mininode_lock: test_node.clear_block_announcement() - inv = CInv(4, block_hash) # 4 == "CompactBlock" + inv = CInv(MSG_CMPCT_BLOCK, block_hash) test_node.send_message(msg_getdata([inv])) wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) @@ -380,7 +380,7 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_on_tip(node, segwit=segwit) if announce == "inv": - test_node.send_message(msg_inv([CInv(2, block.sha256)])) + test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)])) wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock) test_node.send_header_for_blocks([block]) else: @@ -564,7 +564,8 @@ class CompactBlocksTest(BitcoinTestFramework): # We should receive a getdata request test_node.wait_for_getdata([block.sha256], timeout=10) - assert test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG + assert test_node.last_message["getdata"].inv[0].type == MSG_BLOCK or \ + test_node.last_message["getdata"].inv[0].type == MSG_BLOCK | MSG_WITNESS_FLAG # Deliver the block if version == 2: @@ -633,7 +634,7 @@ class CompactBlocksTest(BitcoinTestFramework): wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) test_node.clear_block_announcement() - test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) test_node.clear_block_announcement() @@ -642,7 +643,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.clear_block_announcement() with mininode_lock: test_node.last_message.pop("block", None) - test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) wait_until(lambda: "block" in test_node.last_message, timeout=30, lock=mininode_lock) with mininode_lock: test_node.last_message["block"].block.calc_sha256() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 4f242bd94a..805cb1e84f 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -7,7 +7,7 @@ from decimal import Decimal import time -from test_framework.messages import msg_feefilter +from test_framework.messages import MSG_TX, msg_feefilter from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework @@ -31,7 +31,7 @@ class TestP2PConn(P2PInterface): def on_inv(self, message): for i in message.inv: - if (i.type == 1): + if (i.type == MSG_TX): self.txinvs.append(hashToHex(i.hash)) def clear_invs(self): diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index a8b768c144..15955a938c 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -64,19 +64,40 @@ class FilterTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def run_test(self): - filter_node = self.nodes[0].add_p2p_connection(FilterNode()) - + def test_size_limits(self, filter_node): self.log.info('Check that too large filter is rejected') with self.nodes[0].assert_debug_log(['Misbehaving']): - filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS+1)) - with self.nodes[0].assert_debug_log(['Misbehaving']): filter_node.send_and_ping(msg_filterload(data=b'\xbb'*(MAX_BLOOM_FILTER_SIZE+1))) + self.log.info('Check that max size filter is accepted') + with self.nodes[0].assert_debug_log([], unexpected_msgs=['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xbb'*(MAX_BLOOM_FILTER_SIZE))) + filter_node.send_and_ping(msg_filterclear()) + + self.log.info('Check that filter with too many hash functions is rejected') + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS+1)) + + self.log.info('Check that filter with max hash functions is accepted') + with self.nodes[0].assert_debug_log([], unexpected_msgs=['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS)) + # Don't send filterclear until next two filteradd checks are done + + self.log.info('Check that max size data element to add to the filter is accepted') + with self.nodes[0].assert_debug_log([], unexpected_msgs=['Misbehaving']): + filter_node.send_and_ping(msg_filteradd(data=b'\xcc'*(MAX_SCRIPT_ELEMENT_SIZE))) + self.log.info('Check that too large data element to add to the filter is rejected') with self.nodes[0].assert_debug_log(['Misbehaving']): filter_node.send_and_ping(msg_filteradd(data=b'\xcc'*(MAX_SCRIPT_ELEMENT_SIZE+1))) + filter_node.send_and_ping(msg_filterclear()) + + def run_test(self): + filter_node = self.nodes[0].add_p2p_connection(FilterNode()) + + self.test_size_limits(filter_node) + self.log.info('Add filtered P2P connection to the node') filter_node.send_and_ping(filter_node.watch_filter_init) filter_address = self.nodes[0].decodescript(filter_node.watch_script_pubkey)['addresses'][0] diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py index c9fbb830c8..d743abe681 100755 --- a/test/functional/p2p_fingerprint.py +++ b/test/functional/p2p_fingerprint.py @@ -11,7 +11,7 @@ the node should pretend that it does not have it to avoid fingerprinting. import time from test_framework.blocktools import (create_block, create_coinbase) -from test_framework.messages import CInv +from test_framework.messages import CInv, MSG_BLOCK from test_framework.mininode import ( P2PInterface, msg_headers, @@ -48,7 +48,7 @@ class P2PFingerprintTest(BitcoinTestFramework): # Send a getdata request for a given block hash def send_block_request(self, block_hash, node): msg = msg_getdata() - msg.inv.append(CInv(2, block_hash)) # 2 == "Block" + msg.inv.append(CInv(MSG_BLOCK, block_hash)) node.send_message(msg) # Send a getheaders request for a given single block hash diff --git a/test/functional/p2p_getdata.py b/test/functional/p2p_getdata.py new file mode 100755 index 0000000000..d1b11c2c61 --- /dev/null +++ b/test/functional/p2p_getdata.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test GETDATA processing behavior""" +from collections import defaultdict + +from test_framework.messages import ( + CInv, + msg_getdata, +) +from test_framework.mininode import P2PInterface +from test_framework.test_framework import BitcoinTestFramework + + +class P2PStoreBlock(P2PInterface): + def __init__(self): + super().__init__() + self.blocks = defaultdict(int) + + def on_block(self, message): + message.block.calc_sha256() + self.blocks[message.block.sha256] += 1 + + +class GetdataTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + p2p_block_store = self.nodes[0].add_p2p_connection(P2PStoreBlock()) + + self.log.info("test that an invalid GETDATA doesn't prevent processing of future messages") + + # Send invalid message and verify that node responds to later ping + invalid_getdata = msg_getdata() + invalid_getdata.inv.append(CInv(t=0, h=0)) # INV type 0 is invalid. + p2p_block_store.send_and_ping(invalid_getdata) + + # Check getdata still works by fetching tip block + best_block = int(self.nodes[0].getbestblockhash(), 16) + good_getdata = msg_getdata() + good_getdata.inv.append(CInv(t=2, h=best_block)) + p2p_block_store.send_and_ping(good_getdata) + p2p_block_store.wait_until(lambda: self.nodes[0].p2ps[0].blocks[best_block] == 1) + + +if __name__ == '__main__': + GetdataTest().main() diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 4bd832e8f7..81302374c9 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -7,7 +7,16 @@ import asyncio import struct import sys -from test_framework import messages +from test_framework.messages import ( + CBlockHeader, + CInv, + msg_getdata, + msg_headers, + msg_inv, + msg_ping, + MSG_TX, + ser_string, +) from test_framework.mininode import ( NetworkThread, P2PDataStore, @@ -25,7 +34,7 @@ class msg_unrecognized: self.str_data = str_data.encode() if not isinstance(str_data, bytes) else str_data def serialize(self): - return messages.ser_string(self.str_data) + return ser_string(self.str_data) def __repr__(self): return "{}(data={})".format(self.msgtype, self.str_data) @@ -135,7 +144,7 @@ class InvalidMessagesTest(BitcoinTestFramework): # For some reason unknown to me, we sometimes have to push additional data to the # peer in order for it to realize a disconnect. try: - node.p2p.send_message(messages.msg_ping(nonce=123123)) + node.p2p.send_message(msg_ping(nonce=123123)) except IOError: pass @@ -158,7 +167,7 @@ class InvalidMessagesTest(BitcoinTestFramework): asyncio.run_coroutine_threadsafe(swap_magic_bytes(), NetworkThread.network_event_loop).result() with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART ping']): - conn.send_message(messages.msg_ping(nonce=0xff)) + conn.send_message(msg_ping(nonce=0xff)) conn.wait_for_disconnect(timeout=1) self.nodes[0].disconnect_p2ps() @@ -206,13 +215,13 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_large_inv(self): conn = self.nodes[0].add_p2p_connection(P2PInterface()) with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (0 -> 20): message inv size() = 50001']): - msg = messages.msg_inv([messages.CInv(1, 1)] * 50001) + msg = msg_inv([CInv(MSG_TX, 1)] * 50001) conn.send_and_ping(msg) with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (20 -> 40): message getdata size() = 50001']): - msg = messages.msg_getdata([messages.CInv(1, 1)] * 50001) + msg = msg_getdata([CInv(MSG_TX, 1)] * 50001) conn.send_and_ping(msg) with self.nodes[0].assert_debug_log(['Misbehaving', 'peer=4 (40 -> 60): headers message size = 2001']): - msg = messages.msg_headers([messages.CBlockHeader()] * 2001) + msg = msg_headers([CBlockHeader()] * 2001) conn.send_and_ping(msg) self.nodes[0].disconnect_p2ps() diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 7f7430d04e..157af68203 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -141,12 +141,11 @@ class P2PLeakTest(BitcoinTestFramework): assert no_verack_idlenode.unexpected_msg == False self.log.info('Check that the version message does not leak the local address of the node') - time_begin = int(time.time()) p2p_version_store = self.nodes[0].add_p2p_connection(P2PVersionStore()) - time_end = time.time() ver = p2p_version_store.version_received - assert_greater_than_or_equal(ver.nTime, time_begin) - assert_greater_than_or_equal(time_end, ver.nTime) + # Check that received time is within one hour of now + assert_greater_than_or_equal(ver.nTime, time.time() - 3600) + assert_greater_than_or_equal(time.time() + 3600, ver.nTime) assert_equal(ver.addrFrom.port, 0) assert_equal(ver.addrFrom.ip, '0.0.0.0') assert_equal(ver.nStartingHeight, 201) diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index 6b3436fa5f..da30ad5977 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that we don't leak txs to inbound peers that we haven't yet announced to""" -from test_framework.messages import msg_getdata, CInv +from test_framework.messages import msg_getdata, CInv, MSG_TX from test_framework.mininode import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -37,7 +37,7 @@ class P2PLeakTxTest(BitcoinTestFramework): txid = gen_node.sendtoaddress(gen_node.getnewaddress(), 0.01) want_tx = msg_getdata() - want_tx.inv.append(CInv(t=1, h=int(txid, 16))) + want_tx.inv.append(CInv(t=MSG_TX, h=int(txid, 16))) inbound_peer.last_message.pop('notfound', None) inbound_peer.send_and_ping(want_tx) diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index e6451d9f18..9c8c36c89e 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -8,7 +8,7 @@ Tests that a node configured with -prune=550 signals NODE_NETWORK_LIMITED correc and that it responds to getdata requests for blocks correctly: - send a block within 288 + 2 of the tip - disconnect peers who request blocks older than that.""" -from test_framework.messages import CInv, msg_getdata, msg_verack, NODE_NETWORK_LIMITED, NODE_WITNESS +from test_framework.messages import CInv, MSG_BLOCK, msg_getdata, msg_verack, NODE_NETWORK_LIMITED, NODE_WITNESS from test_framework.mininode import P2PInterface, mininode_lock from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -31,7 +31,7 @@ class P2PIgnoreInv(P2PInterface): wait_until(test_function, timeout=timeout, lock=mininode_lock) def send_getdata_for_block(self, blockhash): getdata_request = msg_getdata() - getdata_request.inv.append(CInv(2, int(blockhash, 16))) + getdata_request.inv.append(CInv(MSG_BLOCK, int(blockhash, 16))) self.send_message(getdata_request) class NodeNetworkLimitedTest(BitcoinTestFramework): diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index dbdce6552a..8a989097b4 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -22,6 +22,8 @@ from test_framework.messages import ( CTxOut, CTxWitness, MAX_BLOCK_BASE_SIZE, + MSG_BLOCK, + MSG_TX, MSG_WITNESS_FLAG, NODE_NETWORK, NODE_WITNESS, @@ -157,7 +159,7 @@ class TestP2PConn(P2PInterface): def announce_tx_and_wait_for_getdata(self, tx, timeout=60, success=True): with mininode_lock: self.last_message.pop("getdata", None) - self.send_message(msg_inv(inv=[CInv(1, tx.sha256)])) + self.send_message(msg_inv(inv=[CInv(MSG_TX, tx.sha256)])) if success: self.wait_for_getdata([tx.sha256], timeout) else: @@ -173,7 +175,7 @@ class TestP2PConn(P2PInterface): if use_header: self.send_message(msg) else: - self.send_message(msg_inv(inv=[CInv(2, block.sha256)])) + self.send_message(msg_inv(inv=[CInv(MSG_BLOCK, block.sha256)])) self.wait_for_getheaders() self.send_message(msg) self.wait_for_getdata([block.sha256]) @@ -576,7 +578,7 @@ class SegWitTest(BitcoinTestFramework): # Verify that if a peer doesn't set nServices to include NODE_WITNESS, # the getdata is just for the non-witness portion. self.old_node.announce_tx_and_wait_for_getdata(tx) - assert self.old_node.last_message["getdata"].inv[0].type == 1 + assert self.old_node.last_message["getdata"].inv[0].type == MSG_TX # Since we haven't delivered the tx yet, inv'ing the same tx from # a witness transaction ought not result in a getdata. @@ -1310,9 +1312,9 @@ class SegWitTest(BitcoinTestFramework): tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program] # Also check that old_node gets a tx announcement, even though this is # a witness transaction. - self.old_node.wait_for_inv([CInv(1, tx2.sha256)]) # wait until tx2 was inv'ed + self.old_node.wait_for_inv([CInv(MSG_TX, tx2.sha256)]) # wait until tx2 was inv'ed test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) - self.old_node.wait_for_inv([CInv(1, tx3.sha256)]) + self.old_node.wait_for_inv([CInv(MSG_TX, tx3.sha256)]) # Test that getrawtransaction returns correct witness information # hash, size, vsize @@ -1896,12 +1898,12 @@ class SegWitTest(BitcoinTestFramework): def test_upgrade_after_activation(self): """Test the behavior of starting up a segwit-aware node after the softfork has activated.""" - # Restart with the new binary self.stop_node(2) self.start_node(2, extra_args=["-segwitheight={}".format(SEGWIT_HEIGHT)]) connect_nodes(self.nodes[0], 2) - self.sync_blocks() + # We reconnect more than 100 blocks, give it plenty of time + self.sync_blocks(timeout=240) # Make sure that this peer thinks segwit has activated. assert softfork_active(self.nodes[2], 'segwit') diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index a8fba306a7..481b1c1841 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -92,6 +92,7 @@ from test_framework.mininode import ( NODE_WITNESS, P2PInterface, mininode_lock, + MSG_BLOCK, msg_block, msg_getblocks, msg_getdata, @@ -120,7 +121,7 @@ class BaseNode(P2PInterface): """Request data for a list of block hashes.""" msg = msg_getdata() for x in block_hashes: - msg.inv.append(CInv(2, x)) + msg.inv.append(CInv(MSG_BLOCK, x)) self.send_message(msg) def send_get_headers(self, locator, hashstop): @@ -131,7 +132,7 @@ class BaseNode(P2PInterface): def send_block_inv(self, blockhash): msg = msg_inv() - msg.inv = [CInv(2, blockhash)] + msg.inv = [CInv(MSG_BLOCK, blockhash)] self.send_message(msg) def send_header_for_blocks(self, new_blocks): diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index a999fba818..10f5eea0e5 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -63,7 +63,7 @@ class TxDownloadTest(BitcoinTestFramework): txid = 0xdeadbeef self.log.info("Announce the txid from each incoming peer to node 0") - msg = msg_inv([CInv(t=1, h=txid)]) + msg = msg_inv([CInv(t=MSG_TX, h=txid)]) for p in self.nodes[0].p2ps: p.send_and_ping(msg) @@ -104,7 +104,7 @@ class TxDownloadTest(BitcoinTestFramework): self.log.info( "Announce the transaction to all nodes from all {} incoming peers, but never send it".format(NUM_INBOUND)) - msg = msg_inv([CInv(t=1, h=txid)]) + msg = msg_inv([CInv(t=MSG_TX, h=txid)]) for p in self.peers: p.send_and_ping(msg) @@ -135,13 +135,13 @@ class TxDownloadTest(BitcoinTestFramework): with mininode_lock: p.tx_getdata_count = 0 - p.send_message(msg_inv([CInv(t=1, h=i) for i in txids])) + p.send_message(msg_inv([CInv(t=MSG_TX, h=i) for i in txids])) wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT, lock=mininode_lock) with mininode_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT) self.log.info("Now check that if we send a NOTFOUND for a transaction, we'll get one more request") - p.send_message(msg_notfound(vec=[CInv(t=1, h=txids[0])])) + p.send_message(msg_notfound(vec=[CInv(t=MSG_TX, h=txids[0])])) wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT + 1, timeout=10, lock=mininode_lock) with mininode_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT + 1) @@ -154,7 +154,7 @@ class TxDownloadTest(BitcoinTestFramework): def test_spurious_notfound(self): self.log.info('Check that spurious notfound is ignored') - self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(1, 1)])) + self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(MSG_TX, 1)])) def run_test(self): # Setup the p2p connections diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 3aaf4b9977..c323168848 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -54,7 +54,7 @@ Node1 is unused in tests 3-7: import time from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script -from test_framework.messages import CBlockHeader, CInv, msg_block, msg_headers, msg_inv +from test_framework.messages import CBlockHeader, CInv, MSG_BLOCK, msg_block, msg_headers, msg_inv from test_framework.mininode import mininode_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -210,7 +210,7 @@ class AcceptBlockTest(BitcoinTestFramework): with mininode_lock: # Clear state so we can check the getdata request test_node.last_message.pop("getdata", None) - test_node.send_message(msg_inv([CInv(2, block_h3.sha256)])) + test_node.send_message(msg_inv([CInv(MSG_BLOCK, block_h3.sha256)])) test_node.sync_with_ping() with mininode_lock: diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index b75ce15f2e..daf02fc4f3 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -20,6 +20,7 @@ import string import configparser import sys + def call_with_auth(node, user, password): url = urllib.parse.urlparse(node.url) headers = {"Authorization": "Basic " + str_to_b64str('{}:{}'.format(user, password))} @@ -64,9 +65,9 @@ class HTTPBasicsTest(BitcoinTestFramework): self.password = lines[3] with open(os.path.join(get_datadir_path(self.options.tmpdir, 0), "bitcoin.conf"), 'a', encoding='utf8') as f: - f.write(rpcauth+"\n") - f.write(rpcauth2+"\n") - f.write(rpcauth3+"\n") + f.write(rpcauth + "\n") + f.write(rpcauth2 + "\n") + f.write(rpcauth3 + "\n") with open(os.path.join(get_datadir_path(self.options.tmpdir, 1), "bitcoin.conf"), 'a', encoding='utf8') as f: f.write("rpcuser={}\n".format(self.rpcuser)) f.write("rpcpassword={}\n".format(self.rpcpassword)) @@ -76,19 +77,16 @@ class HTTPBasicsTest(BitcoinTestFramework): assert_equal(200, call_with_auth(node, user, password).status) self.log.info('Wrong...') - assert_equal(401, call_with_auth(node, user, password+'wrong').status) + assert_equal(401, call_with_auth(node, user, password + 'wrong').status) self.log.info('Wrong...') - assert_equal(401, call_with_auth(node, user+'wrong', password).status) + assert_equal(401, call_with_auth(node, user + 'wrong', password).status) self.log.info('Wrong...') - assert_equal(401, call_with_auth(node, user+'wrong', password+'wrong').status) + assert_equal(401, call_with_auth(node, user + 'wrong', password + 'wrong').status) def run_test(self): - - ################################################## - # Check correctness of the rpcauth config option # - ################################################## + self.log.info('Check correctness of the rpcauth config option') url = urllib.parse.urlparse(self.nodes[0].url) self.test_auth(self.nodes[0], url.username, url.password) @@ -96,12 +94,18 @@ class HTTPBasicsTest(BitcoinTestFramework): self.test_auth(self.nodes[0], 'rt2', self.rt2password) self.test_auth(self.nodes[0], self.user, self.password) - ############################################################### - # Check correctness of the rpcuser/rpcpassword config options # - ############################################################### + self.log.info('Check correctness of the rpcuser/rpcpassword config options') url = urllib.parse.urlparse(self.nodes[1].url) self.test_auth(self.nodes[1], self.rpcuser, self.rpcpassword) + self.log.info('Check that failure to write cookie file will abort the node gracefully') + self.stop_node(0) + cookie_file = os.path.join(get_datadir_path(self.options.tmpdir, 0), self.chain, '.cookie.tmp') + os.mkdir(cookie_file) + init_error = 'Error: Unable to start HTTP server. See debug log for details.' + self.nodes[0].assert_start_raises_init_error(expected_msg=init_error) + + if __name__ == '__main__': - HTTPBasicsTest ().main () + HTTPBasicsTest().main() diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 4855f62a8f..d178e79541 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -54,9 +54,12 @@ NODE_NETWORK_LIMITED = (1 << 10) MSG_TX = 1 MSG_BLOCK = 2 MSG_FILTERED_BLOCK = 3 +MSG_CMPCT_BLOCK = 4 MSG_WITNESS_FLAG = 1 << 30 MSG_TYPE_MASK = 0xffffffff >> 2 +FILTER_TYPE_BASIC = 0 + # Serialization/deserialization tools def sha256(s): return hashlib.new('sha256', s).digest() @@ -1512,3 +1515,103 @@ class msg_no_witness_blocktxn(msg_blocktxn): def serialize(self): return self.block_transactions.serialize(with_witness=False) + +class msg_getcfheaders: + __slots__ = ("filter_type", "start_height", "stop_hash") + msgtype = b"getcfheaders" + + def __init__(self, filter_type, start_height, stop_hash): + self.filter_type = filter_type + self.start_height = start_height + self.stop_hash = stop_hash + + def deserialize(self, f): + self.filter_type = struct.unpack("<B", f.read(1))[0] + self.start_height = struct.unpack("<I", f.read(4))[0] + self.stop_hash = deser_uint256(f) + + def serialize(self): + r = b"" + r += struct.pack("<B", self.filter_type) + r += struct.pack("<I", self.start_height) + r += ser_uint256(self.stop_hash) + return r + + def __repr__(self): + return "msg_getcfheaders(filter_type={:#x}, start_height={}, stop_hash={:x})".format( + self.filter_type, self.start_height, self.stop_hash) + +class msg_cfheaders: + __slots__ = ("filter_type", "stop_hash", "prev_header", "hashes") + msgtype = b"cfheaders" + + def __init__(self, filter_type=None, stop_hash=None, prev_header=None, hashes=None): + self.filter_type = filter_type + self.stop_hash = stop_hash + self.prev_header = prev_header + self.hashes = hashes + + def deserialize(self, f): + self.filter_type = struct.unpack("<B", f.read(1))[0] + self.stop_hash = deser_uint256(f) + self.prev_header = deser_uint256(f) + self.hashes = deser_uint256_vector(f) + + def serialize(self): + r = b"" + r += struct.pack("<B", self.filter_type) + r += ser_uint256(self.stop_hash) + r += ser_uint256(self.prev_header) + r += ser_uint256_vector(self.hashes) + return r + + def __repr__(self): + return "msg_cfheaders(filter_type={:#x}, stop_hash={:x})".format( + self.filter_type, self.stop_hash) + +class msg_getcfcheckpt: + __slots__ = ("filter_type", "stop_hash") + msgtype = b"getcfcheckpt" + + def __init__(self, filter_type, stop_hash): + self.filter_type = filter_type + self.stop_hash = stop_hash + + def deserialize(self, f): + self.filter_type = struct.unpack("<B", f.read(1))[0] + self.stop_hash = deser_uint256(f) + + def serialize(self): + r = b"" + r += struct.pack("<B", self.filter_type) + r += ser_uint256(self.stop_hash) + return r + + def __repr__(self): + return "msg_getcfcheckpt(filter_type={:#x}, stop_hash={:x})".format( + self.filter_type, self.stop_hash) + +class msg_cfcheckpt: + __slots__ = ("filter_type", "stop_hash", "headers") + msgtype = b"cfcheckpt" + + def __init__(self, filter_type=None, stop_hash=None, headers=None): + self.filter_type = filter_type + self.stop_hash = stop_hash + self.headers = headers + + def deserialize(self, f): + self.filter_type = struct.unpack("<B", f.read(1))[0] + self.stop_hash = deser_uint256(f) + self.headers = deser_uint256_vector(f) + + def serialize(self): + r = b"" + r += struct.pack("<B", self.filter_type) + r += ser_uint256(self.stop_hash) + r += ser_uint256_vector(self.headers) + return r + + def __repr__(self): + return "msg_cfcheckpt(filter_type={:#x}, stop_hash={:x})".format( + self.filter_type, self.stop_hash) diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 257499fcb9..d1e982ac3e 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -31,6 +31,8 @@ from test_framework.messages import ( msg_block, MSG_BLOCK, msg_blocktxn, + msg_cfheaders, + msg_cfcheckpt, msg_cmpctblock, msg_feefilter, msg_filteradd, @@ -67,6 +69,8 @@ MESSAGEMAP = { b"addr": msg_addr, b"block": msg_block, b"blocktxn": msg_blocktxn, + b"cfheaders": msg_cfheaders, + b"cfcheckpt": msg_cfcheckpt, b"cmpctblock": msg_cmpctblock, b"feefilter": msg_feefilter, b"filteradd": msg_filteradd, @@ -120,8 +124,9 @@ class P2PConnection(asyncio.Protocol): def is_connected(self): return self._transport is not None - def peer_connect(self, dstaddr, dstport, *, net): + def peer_connect(self, dstaddr, dstport, *, net, timeout_factor): assert not self.is_connected + self.timeout_factor = timeout_factor self.dstaddr = dstaddr self.dstport = dstport # The initial message to send after the connection was made: @@ -327,6 +332,8 @@ class P2PInterface(P2PConnection): def on_addr(self, message): pass def on_block(self, message): pass def on_blocktxn(self, message): pass + def on_cfheaders(self, message): pass + def on_cfcheckpt(self, message): pass def on_cmpctblock(self, message): pass def on_feefilter(self, message): pass def on_filteradd(self, message): pass @@ -367,9 +374,12 @@ class P2PInterface(P2PConnection): # Connection helper methods + def wait_until(self, test_function, timeout=60): + wait_until(test_function, timeout=timeout, lock=mininode_lock, timeout_factor=self.timeout_factor) + def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.is_connected - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) # Message receiving helper methods @@ -380,14 +390,14 @@ class P2PInterface(P2PConnection): return False return self.last_message['tx'].tx.rehash() == txid - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_block(self, blockhash, timeout=60): def test_function(): assert self.is_connected return self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_header(self, blockhash, timeout=60): def test_function(): @@ -397,7 +407,7 @@ class P2PInterface(P2PConnection): return False return last_headers.headers[0].rehash() == int(blockhash, 16) - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_merkleblock(self, blockhash, timeout=60): def test_function(): @@ -407,7 +417,7 @@ class P2PInterface(P2PConnection): return False return last_filtered_block.merkleblock.header.rehash() == int(blockhash, 16) - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_getdata(self, hash_list, timeout=60): """Waits for a getdata message. @@ -421,7 +431,7 @@ class P2PInterface(P2PConnection): return False return [x.hash for x in last_data.inv] == hash_list - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_getheaders(self, timeout=60): """Waits for a getheaders message. @@ -435,7 +445,7 @@ class P2PInterface(P2PConnection): assert self.is_connected return self.last_message.get("getheaders") - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_inv(self, expected_inv, timeout=60): """Waits for an INV message and checks that the first inv object in the message was as expected.""" @@ -448,13 +458,13 @@ class P2PInterface(P2PConnection): self.last_message["inv"].inv[0].type == expected_inv[0].type and \ self.last_message["inv"].inv[0].hash == expected_inv[0].hash - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def wait_for_verack(self, timeout=60): def test_function(): return self.message_count["verack"] - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) # Message sending helper functions @@ -470,7 +480,7 @@ class P2PInterface(P2PConnection): assert self.is_connected return self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) self.ping_counter += 1 @@ -586,7 +596,7 @@ class P2PDataStore(P2PInterface): self.send_message(msg_block(block=b)) else: self.send_message(msg_headers([CBlockHeader(block) for block in blocks])) - wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout, lock=mininode_lock) + self.wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout) if expect_disconnect: self.wait_for_disconnect(timeout=timeout) @@ -594,7 +604,7 @@ class P2PDataStore(P2PInterface): self.sync_with_ping(timeout=timeout) if success: - wait_until(lambda: node.getbestblockhash() == blocks[-1].hash, timeout=timeout) + self.wait_until(lambda: node.getbestblockhash() == blocks[-1].hash, timeout=timeout) else: assert node.getbestblockhash() != blocks[-1].hash @@ -638,6 +648,7 @@ class P2PTxInvStore(P2PInterface): self.tx_invs_received = defaultdict(int) def on_inv(self, message): + super().on_inv(message) # Send getdata in response. # Store how many times invs have been received for each tx. for i in message.inv: if i.type == MSG_TX: @@ -647,3 +658,12 @@ class P2PTxInvStore(P2PInterface): def get_invs(self): with mininode_lock: return list(self.tx_invs_received.keys()) + + def wait_for_broadcast(self, txns, timeout=60): + """Waits for the txns (list of txids) to complete initial broadcast. + The mempool should mark unbroadcast=False for these transactions. + """ + # Wait until invs have been received (and getdatas sent) for each txid. + self.wait_until(lambda: set(self.get_invs()) == set([int(tx, 16) for tx in txns]), timeout) + # Flush messages and wait for the getdatas to be processed + self.sync_with_ping() diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index e475ed8596..9102266456 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -8,6 +8,7 @@ This file is modified from python-bitcoinlib. """ import hashlib import struct +import unittest from .messages import ( CTransaction, @@ -708,3 +709,25 @@ def SegwitV0SignatureHash(script, txTo, inIdx, hashtype, amount): ss += struct.pack("<I", hashtype) return hash256(ss) + +class TestFrameworkScript(unittest.TestCase): + def test_bn2vch(self): + self.assertEqual(bn2vch(0), bytes([])) + self.assertEqual(bn2vch(1), bytes([0x01])) + self.assertEqual(bn2vch(-1), bytes([0x81])) + self.assertEqual(bn2vch(0x7F), bytes([0x7F])) + self.assertEqual(bn2vch(-0x7F), bytes([0xFF])) + self.assertEqual(bn2vch(0x80), bytes([0x80, 0x00])) + self.assertEqual(bn2vch(-0x80), bytes([0x80, 0x80])) + self.assertEqual(bn2vch(0xFF), bytes([0xFF, 0x00])) + self.assertEqual(bn2vch(-0xFF), bytes([0xFF, 0x80])) + self.assertEqual(bn2vch(0x100), bytes([0x00, 0x01])) + self.assertEqual(bn2vch(-0x100), bytes([0x00, 0x81])) + self.assertEqual(bn2vch(0x7FFF), bytes([0xFF, 0x7F])) + self.assertEqual(bn2vch(-0x8000), bytes([0x00, 0x80, 0x80])) + self.assertEqual(bn2vch(-0x7FFFFF), bytes([0xFF, 0xFF, 0xFF])) + self.assertEqual(bn2vch(0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x00])) + self.assertEqual(bn2vch(-0x80000000), bytes([0x00, 0x00, 0x00, 0x80, 0x80])) + self.assertEqual(bn2vch(0xFFFFFFFF), bytes([0xFF, 0xFF, 0xFF, 0xFF, 0x00])) + self.assertEqual(bn2vch(123456789), bytes([0x15, 0xCD, 0x5B, 0x07])) + self.assertEqual(bn2vch(-54321), bytes([0x31, 0xD4, 0x80])) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8719bd0d39..716fa1d845 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -6,11 +6,12 @@ import configparser from enum import Enum -import logging import argparse +import logging import os import pdb import random +import re import shutil import subprocess import sys @@ -101,6 +102,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.bind_to_localhost_only = True self.set_test_params() self.parse_args() + if self.options.timeout_factor == 0 : + self.options.timeout_factor = 99999 + self.rpc_timeout = int(self.rpc_timeout * self.options.timeout_factor) # optionally, increase timeout by a factor def main(self): """Main function. This should not be overridden by the subclass test scripts.""" @@ -136,6 +140,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): sys.exit(exit_code) def parse_args(self): + previous_releases_path = os.getenv("PREVIOUS_RELEASES_DIR") or os.getcwd() + "/releases" parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") @@ -150,6 +155,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="Print out all RPC calls as they are made") parser.add_argument("--portseed", dest="port_seed", default=os.getpid(), type=int, help="The seed to use for assigning port numbers (default: current process id)") + parser.add_argument("--previous-releases", dest="prev_releases", action="store_true", + default=os.path.isdir(previous_releases_path) and bool(os.listdir(previous_releases_path)), + help="Force test of previous releases (default: %(default)s)") parser.add_argument("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") parser.add_argument("--configfile", dest="configfile", @@ -167,8 +175,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="set a random seed for deterministically reproducing a previous test run") parser.add_argument("--descriptors", default=False, action="store_true", help="Run test using a descriptor wallet") + parser.add_argument('--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. Setting it to 0 disables all timeouts') self.add_options(parser) self.options = parser.parse_args() + self.options.previous_releases_path = previous_releases_path def setup(self): """Call this method to start up the test framework object with options set.""" @@ -182,13 +192,22 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): config = configparser.ConfigParser() config.read_file(open(self.options.configfile)) self.config = config - self.options.bitcoind = os.getenv("BITCOIND", default=config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"]) - self.options.bitcoincli = os.getenv("BITCOINCLI", default=config["environment"]["BUILDDIR"] + '/src/bitcoin-cli' + config["environment"]["EXEEXT"]) + fname_bitcoind = os.path.join( + config["environment"]["BUILDDIR"], + "src", + "bitcoind" + config["environment"]["EXEEXT"], + ) + fname_bitcoincli = os.path.join( + config["environment"]["BUILDDIR"], + "src", + "bitcoin-cli" + config["environment"]["EXEEXT"], + ) + self.options.bitcoind = os.getenv("BITCOIND", default=fname_bitcoind) + self.options.bitcoincli = os.getenv("BITCOINCLI", default=fname_bitcoincli) os.environ['PATH'] = os.pathsep.join([ os.path.join(config['environment']['BUILDDIR'], 'src'), - os.path.join(config['environment']['BUILDDIR'], 'src', 'qt'), - os.environ['PATH'] + os.path.join(config['environment']['BUILDDIR'], 'src', 'qt'), os.environ['PATH'] ]) # Set up temp directory and start logging @@ -388,6 +407,25 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): Should only be called once after the nodes have been specified in set_test_params().""" + def get_bin_from_version(version, bin_name, bin_default): + if not version: + return bin_default + return os.path.join( + self.options.previous_releases_path, + re.sub( + r'\.0$', + '', # remove trailing .0 for point releases + 'v{}.{}.{}.{}'.format( + (version % 100000000) // 1000000, + (version % 1000000) // 10000, + (version % 10000) // 100, + (version % 100) // 1, + ), + ), + 'bin', + bin_name, + ) + if self.bind_to_localhost_only: extra_confs = [["bind=127.0.0.1"]] * num_nodes else: @@ -397,9 +435,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if versions is None: versions = [None] * num_nodes if binary is None: - binary = [self.options.bitcoind] * num_nodes + binary = [get_bin_from_version(v, 'bitcoind', self.options.bitcoind) for v in versions] if binary_cli is None: - binary_cli = [self.options.bitcoincli] * num_nodes + binary_cli = [get_bin_from_version(v, 'bitcoin-cli', self.options.bitcoincli) for v in versions] assert_equal(len(extra_confs), num_nodes) assert_equal(len(extra_args), num_nodes) assert_equal(len(versions), num_nodes) @@ -412,6 +450,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): chain=self.chain, rpchost=rpchost, timewait=self.rpc_timeout, + timeout_factor=self.options.timeout_factor, bitcoind=binary[i], bitcoin_cli=binary_cli[i], version=versions[i], @@ -558,6 +597,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args=['-disablewallet'], rpchost=None, timewait=self.rpc_timeout, + timeout_factor=self.options.timeout_factor, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, coverage_dir=None, @@ -640,6 +680,19 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_cli_compiled(): raise SkipTest("bitcoin-cli has not been compiled.") + def skip_if_no_previous_releases(self): + """Skip the running test if previous releases are not available.""" + if not self.has_previous_releases(): + raise SkipTest("previous releases not available or disabled") + + def has_previous_releases(self): + """Checks whether previous releases are present and enabled.""" + if not os.path.isdir(self.options.previous_releases_path): + if self.options.prev_releases: + raise AssertionError("Force test of previous releases but releases missing: {}".format( + self.options.previous_releases_path)) + return self.options.prev_releases + def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" return self.config["components"].getboolean("ENABLE_CLI") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index c0075fd8ec..ebc0501e11 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -62,7 +62,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, chain, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False): + def __init__(self, i, datadir, *, chain, rpchost, timewait, timeout_factor, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -110,7 +110,7 @@ class TestNode(): "--gen-suppressions=all", "--exit-on-first-error=yes", "--error-exitcode=1", "--quiet"] + self.args - if self.version is None or self.version >= 190000: + if self.version_is_at_least(190000): self.args.append("-logthreadnames") self.cli = TestNodeCLI(bitcoin_cli, self.datadir) @@ -128,6 +128,7 @@ class TestNode(): self.perf_subprocesses = {} self.p2ps = [] + self.timeout_factor = timeout_factor AddressKeyPair = collections.namedtuple('AddressKeyPair', ['address', 'key']) PRIV_KEYS = [ @@ -218,9 +219,35 @@ class TestNode(): raise FailedToStartError(self._node_msg( 'bitcoind exited with status {} during initialization'.format(self.process.returncode))) try: - rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.chain, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) + rpc = get_rpc_proxy( + rpc_url(self.datadir, self.index, self.chain, self.rpchost), + self.index, + timeout=self.rpc_timeout // 2, # Shorter timeout to allow for one retry in case of ETIMEDOUT + coveragedir=self.coverage_dir, + ) rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up + if self.version_is_at_least(190000): + # getmempoolinfo.loaded is available since commit + # bb8ae2c (version 0.19.0) + wait_until(lambda: rpc.getmempoolinfo()['loaded']) + # Wait for the node to finish reindex, block import, and + # loading the mempool. Usually importing happens fast or + # even "immediate" when the node is started. However, there + # is no guarantee and sometimes ThreadImport might finish + # later. This is going to cause intermittent test failures, + # because generally the tests assume the node is fully + # ready after being started. + # + # For example, the node will reject block messages from p2p + # when it is still importing with the error "Unexpected + # block message received" + # + # The wait is done here to make tests as robust as possible + # and prevent racy tests and intermittent failures as much + # as possible. Some tests might not need this, but the + # overhead is trivial, and the added guarantees are worth + # the minimal performance cost. self.log.debug("RPC successfully started") if self.use_cli: return @@ -238,7 +265,11 @@ class TestNode(): # succeeds. Try again to properly raise the FailedToStartError pass except OSError as e: - if e.errno != errno.ECONNREFUSED: # Port not yet open? + if e.errno == errno.ETIMEDOUT: + pass # Treat identical to ConnectionResetError + elif e.errno == errno.ECONNREFUSED: + pass # Port not yet open? + else: raise # unknown OS error except ValueError as e: # cookie file not found and no rpcuser or rpcpassword; bitcoind is still starting if "No RPC credentials" not in str(e): @@ -273,6 +304,9 @@ class TestNode(): wallet_path = "wallet/{}".format(urllib.parse.quote(wallet_name)) return RPCOverloadWrapper(self.rpc / wallet_path, descriptors=self.descriptors) + def version_is_at_least(self, ver): + return self.version is None or self.version >= ver + def stop_node(self, expected_stderr='', wait=0): """Stop the node.""" if not self.running: @@ -280,7 +314,7 @@ class TestNode(): self.log.debug("Stopping node") try: # Do not use wait argument when testing older nodes, e.g. in feature_backwards_compatibility.py - if self.version is None or self.version >= 180000: + if self.version_is_at_least(180000): self.stop(wait=wait) else: self.stop() @@ -324,13 +358,13 @@ class TestNode(): return True def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): - wait_until(self.is_node_stopped, timeout=timeout) + wait_until(self.is_node_stopped, timeout=timeout, timeout_factor=self.timeout_factor) @contextlib.contextmanager def assert_debug_log(self, expected_msgs, unexpected_msgs=None, timeout=2): if unexpected_msgs is None: unexpected_msgs = [] - time_end = time.time() + timeout + time_end = time.time() + timeout * self.timeout_factor debug_log = os.path.join(self.datadir, self.chain, 'debug.log') with open(debug_log, encoding='utf-8') as dl: dl.seek(0, 2) @@ -487,7 +521,7 @@ class TestNode(): if 'dstaddr' not in kwargs: kwargs['dstaddr'] = '127.0.0.1' - p2p_conn.peer_connect(**kwargs, net=self.chain)() + p2p_conn.peer_connect(**kwargs, net=self.chain, timeout_factor=self.timeout_factor)() self.p2ps.append(p2p_conn) if wait_for_verack: # Wait for the node to send us the version and verack @@ -501,7 +535,7 @@ class TestNode(): # transaction that will be added to the mempool as soon as we return here. # # So syncing here is redundant when we only want to send a message, but the cost is low (a few milliseconds) - # in comparision to the upside of making tests less fragile and unexpected intermittent errors less likely. + # in comparison to the upside of making tests less fragile and unexpected intermittent errors less likely. p2p_conn.sync_with_ping() return p2p_conn @@ -537,6 +571,8 @@ class TestNodeCLIAttr: def arg_to_cli(arg): if isinstance(arg, bool): return str(arg).lower() + elif arg is None: + return 'null' elif isinstance(arg, dict) or isinstance(arg, list): return json.dumps(arg, default=EncodeDecimal) else: @@ -607,27 +643,13 @@ class RPCOverloadWrapper(): def __getattr__(self, name): return getattr(self.rpc, name) - def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase=None, avoid_reuse=None, descriptors=None): - if self.is_cli: - if disable_private_keys is None: - disable_private_keys = 'null' - if blank is None: - blank = 'null' - if passphrase is None: - passphrase = '' - if avoid_reuse is None: - avoid_reuse = 'null' + def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None): if descriptors is None: descriptors = self.descriptors return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors) def importprivkey(self, privkey, label=None, rescan=None): wallet_info = self.getwalletinfo() - if self.is_cli: - if label is None: - label = 'null' - if rescan is None: - rescan = 'null' if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): return self.__getattr__('importprivkey')(privkey, label, rescan) desc = descsum_create('combo(' + privkey + ')') @@ -642,11 +664,6 @@ class RPCOverloadWrapper(): def addmultisigaddress(self, nrequired, keys, label=None, address_type=None): wallet_info = self.getwalletinfo() - if self.is_cli: - if label is None: - label = 'null' - if address_type is None: - address_type = 'null' if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): return self.__getattr__('addmultisigaddress')(nrequired, keys, label, address_type) cms = self.createmultisig(nrequired, keys, address_type) @@ -662,11 +679,6 @@ class RPCOverloadWrapper(): def importpubkey(self, pubkey, label=None, rescan=None): wallet_info = self.getwalletinfo() - if self.is_cli: - if label is None: - label = 'null' - if rescan is None: - rescan = 'null' if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): return self.__getattr__('importpubkey')(pubkey, label, rescan) desc = descsum_create('combo(' + pubkey + ')') @@ -681,13 +693,6 @@ class RPCOverloadWrapper(): def importaddress(self, address, label=None, rescan=None, p2sh=None): wallet_info = self.getwalletinfo() - if self.is_cli: - if label is None: - label = 'null' - if rescan is None: - rescan = 'null' - if p2sh is None: - p2sh = 'null' if 'descriptors' not in wallet_info or ('descriptors' in wallet_info and not wallet_info['descriptors']): return self.__getattr__('importaddress')(address, label, rescan, p2sh) is_hex = False diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index eb9f6528b3..6dfea7efd2 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -208,9 +208,10 @@ def str_to_b64str(string): def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) -def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None): +def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, timeout_factor=1.0): if attempts == float('inf') and timeout == float('inf'): timeout = 60 + timeout = timeout * timeout_factor attempt = 0 time_end = time.time() + timeout @@ -265,7 +266,7 @@ def get_rpc_proxy(url, node_number, *, timeout=None, coveragedir=None): """ proxy_kwargs = {} if timeout is not None: - proxy_kwargs['timeout'] = timeout + proxy_kwargs['timeout'] = int(timeout) proxy = AuthServiceProxy(url, **proxy_kwargs) proxy.url = url # store URL on proxy for info @@ -398,7 +399,11 @@ def connect_nodes(from_connection, node_num): from_connection.addnode(ip_port, "onetry") # poll until version handshake complete to avoid race conditions # with transaction relaying - wait_until(lambda: all(peer['version'] != 0 for peer in from_connection.getpeerinfo())) + # See comments in net_processing: + # * Must have a version message before anything else + # * Must have a verack message before anything else + wait_until(lambda: all(peer['version'] != 0 for peer in from_connection.getpeerinfo())) + wait_until(lambda: all(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in from_connection.getpeerinfo())) def sync_blocks(rpc_connections, *, wait=1, timeout=60): diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6f5f61de2c..7821355e29 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -24,6 +24,7 @@ import sys import tempfile import re import logging +import unittest # Formatting. Default colors to empty strings. BOLD, GREEN, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") @@ -65,6 +66,10 @@ if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 +TEST_FRAMEWORK_MODULES = [ + "script", +] + EXTENDED_SCRIPTS = [ # These tests are not run by default. # Longest test should go first, to favor running tests in parallel @@ -152,6 +157,7 @@ BASE_SCRIPTS = [ 'rpc_deprecated.py', 'wallet_disable.py', 'p2p_addr_relay.py', + 'p2p_getdata.py', 'rpc_net.py', 'wallet_keypool.py', 'wallet_keypool.py --descriptors', @@ -220,6 +226,7 @@ BASE_SCRIPTS = [ 'feature_loadblock.py', 'p2p_dos_header_tree.py', 'p2p_unrequested_blocks.py', + 'p2p_blockfilters.py', 'feature_includeconf.py', 'feature_asmap.py', 'mempool_unbroadcast.py', @@ -237,7 +244,6 @@ BASE_SCRIPTS = [ 'rpc_help.py', 'feature_help.py', 'feature_shutdown.py', - 'framework_test_script.py', # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] @@ -400,6 +406,16 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= if os.path.isdir(cache_dir): print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir)) + # Test Framework Tests + print("Running Unit Tests for Test Framework Modules") + test_framework_tests = unittest.TestSuite() + for module in TEST_FRAMEWORK_MODULES: + test_framework_tests.addTest(unittest.TestLoader().loadTestsFromName("test_framework.{}".format(module))) + result = unittest.TextTestRunner(verbosity=1, failfast=True).run(test_framework_tests) + if not result.wasSuccessful(): + logging.debug("Early exiting after failure in TestFramework unit tests") + sys.exit(False) + tests_dir = src_dir + '/test/functional/' flags = ['--cachedir={}'.format(cache_dir)] + args @@ -623,7 +639,7 @@ class TestResult(): def check_script_prefixes(): """Check that test scripts start with one of the allowed name prefixes.""" - good_prefixes_re = re.compile("^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool|framework_test)_") + good_prefixes_re = re.compile("^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_") bad_script_names = [script for script in ALL_SCRIPTS if good_prefixes_re.match(script) is None] if bad_script_names: diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index b3d496dd51..524e1593ba 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -15,6 +15,7 @@ from test_framework.util import assert_equal BUFFER_SIZE = 16 * 1024 + class ToolWalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -48,7 +49,7 @@ class ToolWalletTest(BitcoinTestFramework): h = hashlib.sha1() mv = memoryview(bytearray(BUFFER_SIZE)) with open(self.wallet_path, 'rb', buffering=0) as f: - for n in iter(lambda : f.readinto(mv), 0): + for n in iter(lambda: f.readinto(mv), 0): h.update(mv[:n]) return h.hexdigest() @@ -69,7 +70,12 @@ class ToolWalletTest(BitcoinTestFramework): self.assert_raises_tool_error('Invalid command: help', 'help') self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') - self.assert_raises_tool_error('Error loading wallet.dat. Is wallet being used by other process?', '-wallet=wallet.dat', 'info') + self.assert_raises_tool_error( + 'Error initializing wallet database environment "{}"!\nError loading wallet.dat. Is wallet being used by other process?' + .format(os.path.join(self.nodes[0].datadir, self.chain, 'wallets')), + '-wallet=wallet.dat', + 'info', + ) self.assert_raises_tool_error('Error: no wallet file at nonexistent.dat', '-wallet=nonexistent.dat', 'info') def test_tool_wallet_info(self): @@ -84,7 +90,7 @@ class ToolWalletTest(BitcoinTestFramework): # # self.log.debug('Setting wallet file permissions to 400 (read-only)') # os.chmod(self.wallet_path, stat.S_IRUSR) - # assert(self.wallet_permissions() in ['400', '666']) # Sanity check. 666 because Appveyor. + # assert self.wallet_permissions() in ['400', '666'] # Sanity check. 666 because Appveyor. # shasum_before = self.wallet_shasum() timestamp_before = self.wallet_timestamp() self.log.debug('Wallet file timestamp before calling info: {}'.format(timestamp_before)) @@ -103,7 +109,7 @@ class ToolWalletTest(BitcoinTestFramework): self.log_wallet_timestamp_comparison(timestamp_before, timestamp_after) self.log.debug('Setting wallet file permissions back to 600 (read/write)') os.chmod(self.wallet_path, stat.S_IRUSR | stat.S_IWUSR) - assert(self.wallet_permissions() in ['600', '666']) # Sanity check. 666 because Appveyor. + assert self.wallet_permissions() in ['600', '666'] # Sanity check. 666 because Appveyor. # # TODO: Wallet tool info should not write to the wallet file. # The following lines should be uncommented and the tests still succeed: @@ -197,6 +203,14 @@ class ToolWalletTest(BitcoinTestFramework): assert_equal(shasum_after, shasum_before) self.log.debug('Wallet file shasum unchanged\n') + def test_salvage(self): + # TODO: Check salvage actually salvages and doesn't break things. https://github.com/bitcoin/bitcoin/issues/7463 + self.log.info('Check salvage') + self.start_node(0, ['-wallet=salvage']) + self.stop_node(0) + + self.assert_tool_output('', '-wallet=salvage', 'salvage') + def run_test(self): self.wallet_path = os.path.join(self.nodes[0].datadir, self.chain, 'wallets', 'wallet.dat') self.test_invalid_tool_commands_and_args() @@ -205,7 +219,7 @@ class ToolWalletTest(BitcoinTestFramework): self.test_tool_wallet_info_after_transaction() self.test_tool_wallet_create_on_existing_wallet() self.test_getwalletinfo_on_different_wallet() - + self.test_salvage() if __name__ == '__main__': ToolWalletTest().main() diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index c7b19081c8..90d17a806c 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -18,7 +18,6 @@ from test_framework.util import ( assert_raises_rpc_error, connect_nodes, disconnect_nodes, - wait_until, ) @@ -98,7 +97,7 @@ class AbandonConflictTest(BitcoinTestFramework): # TODO: redo with eviction self.stop_node(0) self.start_node(0, extra_args=["-minrelaytxfee=0.0001"]) - wait_until(lambda: self.nodes[0].getmempoolinfo()['loaded']) + assert self.nodes[0].getmempoolinfo()['loaded'] # Verify txs no longer in either node's mempool assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -126,7 +125,7 @@ class AbandonConflictTest(BitcoinTestFramework): # Verify that even with a low min relay fee, the tx is not reaccepted from wallet on startup once abandoned self.stop_node(0) self.start_node(0, extra_args=["-minrelaytxfee=0.00001"]) - wait_until(lambda: self.nodes[0].getmempoolinfo()['loaded']) + assert self.nodes[0].getmempoolinfo()['loaded'] assert_equal(len(self.nodes[0].getrawmempool()), 0) assert_equal(self.nodes[0].getbalance(), balance) @@ -148,7 +147,7 @@ class AbandonConflictTest(BitcoinTestFramework): # Remove using high relay fee again self.stop_node(0) self.start_node(0, extra_args=["-minrelaytxfee=0.0001"]) - wait_until(lambda: self.nodes[0].getmempoolinfo()['loaded']) + assert self.nodes[0].getmempoolinfo()['loaded'] assert_equal(len(self.nodes[0].getrawmempool()), 0) newbalance = self.nodes[0].getbalance() assert_equal(newbalance, balance - Decimal("24.9996")) diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 2dddbf2cf3..797c903dd3 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet.""" from decimal import Decimal -import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -405,8 +404,6 @@ class WalletTest(BitcoinTestFramework): '-reindex', '-zapwallettxes=1', '-zapwallettxes=2', - # disabled until issue is fixed: https://github.com/bitcoin/bitcoin/issues/7463 - # '-salvagewallet', ] chainlimit = 6 for m in maintenance: @@ -466,12 +463,8 @@ class WalletTest(BitcoinTestFramework): extra_args = ["-walletrejectlongchains", "-limitancestorcount=" + str(2 * chainlimit)] self.start_node(0, extra_args=extra_args) - # wait for loadmempool - timeout = 10 - while (timeout > 0 and len(self.nodes[0].getrawmempool()) < chainlimit * 2): - time.sleep(0.5) - timeout -= 0.5 - assert_equal(len(self.nodes[0].getrawmempool()), chainlimit * 2) + # wait until the wallet has submitted all transactions to the mempool + wait_until(lambda: len(self.nodes[0].getrawmempool()) == chainlimit * 2) node0_balance = self.nodes[0].getbalance() # With walletrejectlongchains we will not create the tx and store it in our wallet. diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 9ba23db42d..27197e3b6d 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -300,7 +300,7 @@ def test_maxtxfee_fails(self, rbf_node, dest_address): self.restart_node(1, ['-maxtxfee=0.000025'] + self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) rbfid = spend_one_input(rbf_node, dest_address) - assert_raises_rpc_error(-4, "Unable to create transaction: Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) + assert_raises_rpc_error(-4, "Unable to create transaction. Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) self.restart_node(1, self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) @@ -517,7 +517,7 @@ def test_no_more_inputs_fails(self, rbf_node, dest_address): rbf_node.generatetoaddress(1, dest_address) # spend all funds, no change output rbfid = rbf_node.sendtoaddress(rbf_node.getnewaddress(), rbf_node.getbalance(), "", "", True) - assert_raises_rpc_error(-4, "Unable to create transaction: Insufficient funds", rbf_node.bumpfee, rbfid) + assert_raises_rpc_error(-4, "Unable to create transaction. Insufficient funds", rbf_node.bumpfee, rbfid) if __name__ == "__main__": diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 09f89eb59d..5b083a5398 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -170,5 +170,101 @@ class WalletHDTest(BitcoinTestFramework): assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, new_seed) assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress())) + self.log.info('Test sethdseed restoring with keys outside of the initial keypool') + self.nodes[0].generate(10) + # Restart node 1 with keypool of 3 and a different wallet + self.nodes[1].createwallet(wallet_name='origin', blank=True) + self.stop_node(1) + self.start_node(1, extra_args=['-keypool=3', '-wallet=origin']) + connect_nodes(self.nodes[0], 1) + + # sethdseed restoring and seeing txs to addresses out of the keypool + origin_rpc = self.nodes[1].get_wallet_rpc('origin') + seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress()) + origin_rpc.sethdseed(True, seed) + + self.nodes[1].createwallet(wallet_name='restore', blank=True) + restore_rpc = self.nodes[1].get_wallet_rpc('restore') + restore_rpc.sethdseed(True, seed) # Set to be the same seed as origin_rpc + restore_rpc.sethdseed(True) # Rotate to a new seed, making original `seed` inactive + + self.nodes[1].createwallet(wallet_name='restore2', blank=True) + restore2_rpc = self.nodes[1].get_wallet_rpc('restore2') + restore2_rpc.sethdseed(True, seed) # Set to be the same seed as origin_rpc + restore2_rpc.sethdseed(True) # Rotate to a new seed, making original `seed` inactive + + # Check persistence of inactive seed by reloading restore. restore2 is still loaded to test the case where the wallet is not reloaded + restore_rpc.unloadwallet() + self.nodes[1].loadwallet('restore') + restore_rpc = self.nodes[1].get_wallet_rpc('restore') + + # Empty origin keypool and get an address that is beyond the initial keypool + origin_rpc.getnewaddress() + origin_rpc.getnewaddress() + last_addr = origin_rpc.getnewaddress() # Last address of initial keypool + addr = origin_rpc.getnewaddress() # First address beyond initial keypool + + # Check that the restored seed has last_addr but does not have addr + info = restore_rpc.getaddressinfo(last_addr) + assert_equal(info['ismine'], True) + info = restore_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], False) + info = restore2_rpc.getaddressinfo(last_addr) + assert_equal(info['ismine'], True) + info = restore2_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], False) + # Check that the origin seed has addr + info = origin_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], True) + + # Send a transaction to addr, which is out of the initial keypool. + # The wallet that has set a new seed (restore_rpc) should not detect this transaction. + txid = self.nodes[0].sendtoaddress(addr, 1) + origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex']) + self.nodes[0].generate(1) + origin_rpc.gettransaction(txid) + assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, txid) + out_of_kp_txid = txid + + # Send a transaction to last_addr, which is in the initial keypool. + # The wallet that has set a new seed (restore_rpc) should detect this transaction and generate 3 new keys from the initial seed. + # The previous transaction (out_of_kp_txid) should still not be detected as a rescan is required. + txid = self.nodes[0].sendtoaddress(last_addr, 1) + origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex']) + self.nodes[0].generate(1) + origin_rpc.gettransaction(txid) + restore_rpc.gettransaction(txid) + assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, out_of_kp_txid) + restore2_rpc.gettransaction(txid) + assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore2_rpc.gettransaction, out_of_kp_txid) + + # After rescanning, restore_rpc should now see out_of_kp_txid and generate an additional key. + # addr should now be part of restore_rpc and be ismine + restore_rpc.rescanblockchain() + restore_rpc.gettransaction(out_of_kp_txid) + info = restore_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], True) + restore2_rpc.rescanblockchain() + restore2_rpc.gettransaction(out_of_kp_txid) + info = restore2_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], True) + + # Check again that 3 keys were derived. + # Empty keypool and get an address that is beyond the initial keypool + origin_rpc.getnewaddress() + origin_rpc.getnewaddress() + last_addr = origin_rpc.getnewaddress() + addr = origin_rpc.getnewaddress() + + # Check that the restored seed has last_addr but does not have addr + info = restore_rpc.getaddressinfo(last_addr) + assert_equal(info['ismine'], True) + info = restore_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], False) + info = restore2_rpc.getaddressinfo(last_addr) + assert_equal(info['ismine'], True) + info = restore2_rpc.getaddressinfo(addr) + assert_equal(info['ismine'], False) + if __name__ == '__main__': WalletHDTest().main () diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index c569416292..ff9ff34185 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -122,10 +122,6 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(['-zapwallettxes=1', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file") self.nodes[0].assert_start_raises_init_error(['-zapwallettxes=2', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file") - self.log.info("Do not allow -salvagewallet with multiwallet") - self.nodes[0].assert_start_raises_init_error(['-salvagewallet', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file") - self.nodes[0].assert_start_raises_init_error(['-salvagewallet=1', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file") - # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) @@ -227,10 +223,10 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-18, 'Wallet wallets not found.', self.nodes[0].loadwallet, 'wallets') # Fail to load duplicate wallets - assert_raises_rpc_error(-4, 'Wallet file verification failed: Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0]) + assert_raises_rpc_error(-4, 'Wallet file verification failed. Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0]) # Fail to load duplicate wallets by different ways (directory and filepath) - assert_raises_rpc_error(-4, "Wallet file verification failed: Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') + assert_raises_rpc_error(-4, "Wallet file verification failed. Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') # Fail to load if one wallet is a copy of another assert_raises_rpc_error(-4, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') @@ -240,7 +236,7 @@ class MultiWalletTest(BitcoinTestFramework): # Fail to load if wallet file is a symlink - assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') + assert_raises_rpc_error(-4, "Wallet file verification failed. Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink') # Fail to load if a directory is specified that doesn't contain a wallet os.mkdir(wallet_dir('empty_wallet_dir')) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index d04bc4ce44..bb81746715 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -4,22 +4,25 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """upgradewallet RPC functional test -Test upgradewallet RPC. Download v0.15.2 v0.16.3 node binaries: +Test upgradewallet RPC. Download node binaries: -contrib/devtools/previous_release.sh -b v0.15.2 v0.16.3 +contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 + +Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py """ import os import shutil -from test_framework.test_framework import BitcoinTestFramework, SkipTest +from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( adjust_bitcoin_conf_for_pre_17, assert_equal, assert_greater_than, - assert_is_hex_string + assert_is_hex_string, ) + class UpgradeWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -32,32 +35,16 @@ class UpgradeWalletTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_previous_releases() def setup_network(self): self.setup_nodes() def setup_nodes(self): - if os.getenv("TEST_PREVIOUS_RELEASES") == "false": - raise SkipTest("upgradewallet RPC tests") - - releases_path = os.getenv("PREVIOUS_RELEASES_DIR") or os.getcwd() + "/releases" - if not os.path.isdir(releases_path): - if os.getenv("TEST_PREVIOUS_RELEASES") == "true": - raise AssertionError("TEST_PREVIOUS_RELEASES=1 but releases missing: " + releases_path) - raise SkipTest("This test requires binaries for previous releases") - self.add_nodes(self.num_nodes, extra_args=self.extra_args, versions=[ None, 160300, - 150200 - ], binary=[ - self.options.bitcoind, - releases_path + "/v0.16.3/bin/bitcoind", - releases_path + "/v0.15.2/bin/bitcoind", - ], binary_cli=[ - self.options.bitcoincli, - releases_path + "/v0.16.3/bin/bitcoin-cli", - releases_path + "/v0.15.2/bin/bitcoin-cli", + 150200, ]) # adapt bitcoin.conf, because older bitcoind's don't recognize config sections adjust_bitcoin_conf_for_pre_17(self.nodes[1].bitcoinconf) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index e2454c4237..56b18752ec 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -38,6 +38,7 @@ def main(): ) parser.add_argument( '--par', + '-j', type=int, default=4, help='How many targets to merge or execute in parallel.', diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 2bb76ec286..563e076b35 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -35,8 +35,9 @@ if ! command -v shellcheck > /dev/null; then exit $EXIT_CODE fi +SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced) EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" -if ! shellcheck "$EXCLUDE" $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then +if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then EXIT_CODE=1 fi @@ -46,14 +47,13 @@ if ! command -v yq > /dev/null; then fi EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")" -SHELLCHECK_CMD="shellcheck --external-sources --check-sourced $EXCLUDE_GITIAN" for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml') do script=$(basename "$descriptor") # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script. echo "#!/bin/bash" > $script yq -r .script "$descriptor" >> $script - if ! $SHELLCHECK_CMD $script; then + if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE_GITIAN" $script; then EXIT_CODE=1 fi rm $script |